From 0fcab6a9758f52e588d5a94c7da91b0ed5f76618 Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Tue, 14 Feb 2023 16:35:39 +0100 Subject: [PATCH 1/7] smt smt --- build/docker-compose.yml | 26 +- build/grafana/config.yml | 6 + build/grafana/dashboards/live-network.json | 1959 ------------------- compositions/cluster-k8s/arabica/flood.toml | 10 +- manifest.toml | 2 + tests/helpers/fund-accs/run_light.go | 13 +- 6 files changed, 32 insertions(+), 1984 deletions(-) create mode 100644 build/grafana/config.yml delete mode 100644 build/grafana/dashboards/live-network.json diff --git a/build/docker-compose.yml b/build/docker-compose.yml index a644a704..fec5d1fe 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -1,27 +1,14 @@ version: '3.7' services: - loki: - container_name: loki - image: grafana/loki:2.6.1 - expose: - - 3100 + grafana: + image: grafana/grafana:7.5.7 ports: - - "3100:3100" + - 3000:3000 restart: unless-stopped volumes: - - loki-data:/loki - - promtail: - container_name: promtail - image: grafana/promtail:2.6.1 - volumes: - # custom config will read logs from the containers of - # this project - - ${PWD}/promtail:/etc/promtail - # to read container labels and logs - - /var/run/docker.sock:/var/run/docker.sock - - /var/lib/docker/containers:/var/lib/docker/containers + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + - grafana-data:/var/lib/grafana prometheus: container_name: prometheus @@ -49,4 +36,5 @@ services: volumes: prometheus-data: - loki-data: \ No newline at end of file + loki-data: + grafana-data: \ No newline at end of file diff --git a/build/grafana/config.yml b/build/grafana/config.yml new file mode 100644 index 00000000..2d8ca1db --- /dev/null +++ b/build/grafana/config.yml @@ -0,0 +1,6 @@ +datasources: + - name: Prometheus + access: proxy + type: prometheus + url: http://0.0.0.0:9090 + isDefault: true \ No newline at end of file diff --git a/build/grafana/dashboards/live-network.json b/build/grafana/dashboards/live-network.json deleted file mode 100644 index 2f97d993..00000000 --- a/build/grafana/dashboards/live-network.json +++ /dev/null @@ -1,1959 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 0 - }, - "id": 123156, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "count(count by (exported_instance) (testground_das_network_head))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "# Light Node Alive", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 2, - "y": 0 - }, - "id": 123163, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "testground_head", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Network Head", - "transformations": [ - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Time" - } - ] - } - }, - { - "id": "reduce", - "options": { - "reducers": [ - "max" - ] - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 18, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 4, - "y": 0 - }, - "id": 123145, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "vertical", - "showValue": "always", - "stacking": "normal", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head)", - "format": "table", - "instant": true, - "interval": "5m", - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "# nodes / height", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "testground_das_sampled_chain_head": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Value": 0, - "testground_das_sampled_chain_head": 1 - }, - "renameByName": { - "Value": "Count", - "testground_das_sampled_chain_head": "Height" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Count": { - "aggregations": [], - "operation": "groupby" - }, - "Height": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": false, - "field": "Height" - } - ] - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 15, - "y": 0 - }, - "id": 123144, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "never", - "stacking": "normal", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head)", - "format": "table", - "instant": true, - "interval": "5m", - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "# nodes per height", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "testground_das_sampled_chain_head": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Value": "Count", - "testground_das_sampled_chain_head": "Height" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Count": { - "aggregations": [], - "operation": "groupby" - }, - "Height": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Percentage", - "binary": { - "left": "Count", - "operator": "/", - "reducer": "sum", - "right": "500" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Count" - } - ] - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 0, - "y": 7 - }, - "id": 123153, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "testground_node_share_blackbox_eds_size_sum", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "EDS Size", - "transformations": [ - { - "id": "groupBy", - "options": {} - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 3, - "y": 7 - }, - "id": 123152, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(testground_node_share_blackbox_eds_size_sum)/2", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Square Size", - "transformations": [ - { - "id": "groupBy", - "options": {} - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "% of DASing Requests =< 500 (ms)", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 6, - "y": 7 - }, - "id": 123129, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"$light_node_id\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"}))", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "% of DAS Requests =< 500 (ms)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "% of DASing Requests > 1s", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 9, - "y": 7 - }, - "id": 123134, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"})) - (sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"$light_node_id\", le=\"1\"}))", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "% of DAS Requests > 1s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1000 - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 12, - "y": 7 - }, - "id": 123161, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "avg by (height) (testground_das_sample_time_hist_sum{exported_instance=\"$light_node_id\"}/testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"})", - "format": "heatmap", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Avg DASing time (ms)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 15, - "y": 7 - }, - "id": 123137, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "auto", - "showValue": "always", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg by (header) (testground_das_sample_time_hist_sum{exported_instance=\"$light_node_id\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "DASing time per Height (current instance)", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "header": { - "aggregations": [], - "operation": "groupby" - } - } - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 123142, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "colorByField": "exported_instance", - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "auto", - "showValue": "always", - "stacking": "normal", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "header", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 100 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg by (header, exported_instance) (testground_das_sample_time_hist_sum)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "DASing time overview (all instances)", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "exported_instance": { - "aggregations": [], - "operation": "groupby" - }, - "header": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": false, - "field": "header" - } - ] - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 22 - }, - "id": 123133, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "(rate(testground_node_header_blackbox_request_duration_sum{exported_instance=\"$light_node_id\"}[$__range])/rate(testground_node_header_blackbox_request_duration_count{exported_instance=\"$light_node_id\"}[$__range]))/1000", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Avg Header Transfer Rate (bytes/s)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 22 - }, - "id": 123131, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "(rate(testground_node_share_blackbox_request_duration_sum{exported_instance=\"$light_node_id\"}[$__range])/rate(testground_node_share_blackbox_request_duration_count{exported_instance=\"$light_node_id\"}[$__range]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Avg Share Transfer Rate (bytes/s)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 22 - }, - "id": 123148, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_bandwidth_rate_outbound_sum{exported_instance=\"$light_node_id\"}/testground_p2p_bandwidth_rate_outbound_count{exported_instance=\"$light_node_id\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Avg OUT BW Rate (bytes/s)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 22 - }, - "id": 123150, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_bandwidth_rate_inbound_sum{exported_instance=\"$light_node_id\"}/testground_p2p_bandwidth_rate_inbound_count{exported_instance=\"$light_node_id\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Avg IN BW Rate (bytes/s)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 123149, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_bandwidth_total_outbound_sum{exported_instance=\"$light_node_id\"}/testground_p2p_bandwidth_total_outbound_count{exported_instance=\"$light_node_id\"}", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total OUT BW", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 123159, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_bandwidth_total_inbound_sum{exported_instance=\"$light_node_id\"}/testground_p2p_bandwidth_total_inbound_count{exported_instance=\"$light_node_id\"}", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total IN BW", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 0, - "y": 37 - }, - "id": 123158, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "count(count by (peer_id) (testground_p2p_total_inbound_by_peer_sum{exported_instance=\"$light_node_id\"}))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "# Peers Of Current Instance", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 3, - "y": 37 - }, - "id": 123147, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_total_outbound_by_peer_sum{exported_instance=\"$light_node_id\"}/testground_p2p_total_outbound_by_peer_count{exported_instance=\"$light_node_id\"}", - "format": "time_series", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total OUT BW by Peer", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 14, - "y": 37 - }, - "id": 123160, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "testground_p2p_total_inbound_by_peer_sum{exported_instance=\"$light_node_id\"}/testground_p2p_total_inbound_by_peer_count{exported_instance=\"$light_node_id\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total IN BW by Peer", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 123127, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "(avg by (height) (testground_node_header_blackbox_block_time_sum{exported_instance=\"$light_node_id\"}))/1000", - "format": "table", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Block time per Height", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "height": { - "aggregations": [], - "operation": "groupby" - } - } - } - } - ], - "type": "barchart" - } - ], - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "12D3KooWF9PnoopchmXBC7RyRFDQyJXRbP218Fidoa3qR8RWqdhR", - "value": "12D3KooWF9PnoopchmXBC7RyRFDQyJXRbP218Fidoa3qR8RWqdhR" - }, - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "definition": "label_values(testground_das_sample_time_hist_sum, exported_instance)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "light_node_id", - "options": [], - "query": { - "query": "label_values(testground_das_sample_time_hist_sum, exported_instance)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "influxdb", - "uid": "CIHiCGKVk" - }, - "definition": "show tag values with key = \"run\"", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "run", - "options": [], - "query": "show tag values with key = \"run\"", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "hidden": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "type": "timepicker" - }, - "timezone": "browser", - "title": "Live Network Light Nodes Metrics -", - "uid": "Fccs8xxTacVj", - "version": 17, - "weekStart": "" -} \ No newline at end of file diff --git a/compositions/cluster-k8s/arabica/flood.toml b/compositions/cluster-k8s/arabica/flood.toml index 2b1bfc44..7d1cfbe3 100644 --- a/compositions/cluster-k8s/arabica/flood.toml +++ b/compositions/cluster-k8s/arabica/flood.toml @@ -5,7 +5,7 @@ [global] plan = "celestia" case = "flood-arabica" - total_instances = 140 + total_instances = 160 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -13,17 +13,17 @@ artifact = "" [global.run.test_params] execution-time = "20" - light = "140" + light = "160" [[groups]] id = "lights" builder = "docker:generic" [groups.resources] - memory = "300Mi" - cpu = "200m" + memory = "400Mi" + cpu = "250m" [groups.instances] - count = 140 + count = 160 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" diff --git a/manifest.toml b/manifest.toml index 2c5c1214..30b29444 100644 --- a/manifest.toml +++ b/manifest.toml @@ -125,6 +125,7 @@ instances = { min = 4, max = 3000, default = 12 } block-height = { type = "int" } role = { type = "string" } p2p-network = { type = "string", default = "private" } + otel-collector-address = { type = "string", default = "46.101.22.123:4318" } [[testcases]] name = "get-shares-by-namespace" @@ -181,6 +182,7 @@ instances = { min = 30, max = 1002, default = 30 } block-height = { type = "int" } role = { type = "string" } p2p-network = { type = "string", default = "private" } + otel-collector-address = { type = "string", default = "46.101.22.123:4318" } [[testcases]] name = "flood-arabica" diff --git a/tests/helpers/fund-accs/run_light.go b/tests/helpers/fund-accs/run_light.go index 23e21b95..8f052981 100644 --- a/tests/helpers/fund-accs/run_light.go +++ b/tests/helpers/fund-accs/run_light.go @@ -3,6 +3,8 @@ package fundaccounts import ( "context" "fmt" + "github.com/celestiaorg/celestia-node/nodebuilder" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "time" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -86,7 +88,16 @@ func RunLightNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { cfg.Gateway.Enabled = true cfg.Gateway.Port = "26659" - nd, err := nodekit.NewNode(ndhome, node.Light, runenv.StringParam("p2p-network"), cfg) + optlOpts := []otlpmetrichttp.Option{ + otlpmetrichttp.WithEndpoint(runenv.StringParam("otel-collector-address")), + otlpmetrichttp.WithInsecure(), + } + nd, err := nodekit.NewNode(ndhome, node.Light, runenv.StringParam("p2p-network"), cfg, + nodebuilder.WithMetrics( + optlOpts, + node.Light, + ), + nodebuilder.WithBlackboxMetrics()) if err != nil { return err } From 94817b8f2e7da5763ab13a4ce53172ffd308e9d0 Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 08:07:59 +0100 Subject: [PATCH 2/7] try flood with nodelogger --- build/.env | 59 ++++++++++++++++++++ build/docker-compose.yml | 78 ++++++++++++++++++++++++++- tests/helpers/fund-accs/run_bridge.go | 4 +- tests/helpers/fund-accs/run_light.go | 1 + 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 build/.env diff --git a/build/.env b/build/.env new file mode 100644 index 00000000..f735b28e --- /dev/null +++ b/build/.env @@ -0,0 +1,59 @@ +# OTEL collector +OTEL_GRPC_PORT="4317" +OTEL_HTTP_PORT="4318" +OTEL_PROMETHEUS_EXPORTER="8889" + +# Prometheus +PROMETHEUS_PORT="9090" +PROMETHEUS_URL="http://46.101.22.123:${PROMETHEUS_PORT}" +# PROMETHEUS_URL="https://:@prometheus-prod-10-prod-us-central-0.grafana.net/api/prom" + +# Celestia Core/App +DENOM="utia" +VALIDATOR_KEY="validator" +CHAINID="private" +KEYRING_BACKEND="test" +APP_HOME_DIR="/root/.celestia-app" +CORE_IP="celestia-app" +CORE_RPC_PORT="26657" +CORE_GRPC_PORT="9092" +APP_NODE_GRPC_TLS="false" + +NODE_EXPORTED_KEY_FILE_POSTFIX="_nodeKey.txt" + +# Valuter +VALUTER_PORT=8080 +VALUTER_SERVING_ADDR=:${VALUTER_PORT} + +# Database +POSTGRES_DB=celestiadb +POSTGRES_USER=root +POSTGRES_PASSWORD=password +POSTGRES_PORT=5432 +POSTGRES_HOST=postgres +PGADMIN_PORT=5555 + + +# Nodelogger +NODELOGGER_LOG_LEVEL="info" +# in seconds +NODELOGGER_PROMETHEUS_SYNC_INTERVAL=30 +NODELOGGER_API_ROWS_PER_PAGE=100 +NODELOGGER_REST_API_PORT="5050" +NODELOGGER_REST_API_ADDRESS=":${NODELOGGER_REST_API_PORT}" +NODELOGGER_DEMO="false" +NODELOGGER_ORIGIN_ALLOWED="*" + +# Leaderboard +LEADERBOARD_LOG_LEVEL="info" +LEADERBOARD_API_CALL_RETRY=20 +LEADERBOARD_API_CALL_TIMEOUT=60 +LEADERBOARD_PROMETHEUS_SYNC_INTERVAL=30 +LEADERBOARD_VALIDATOR_SYNC_INTERVAL=60 +LEADERBOARD_TENDERMINT_SYNC_INTERVAL=86400 +LEADERBOARD_API_ROWS_PER_PAGE=100 +LEADERBOARD_REST_API_PORT="5051" +LEADERBOARD_REST_API_ADDRESS=":${LEADERBOARD_REST_API_PORT}" +LEADERBOARD_IP_INFO_API_KEY="xxxxxxxxxxxx" +LEADERBOARD_DEMO="false" +LEADERBOARD_ORIGIN_ALLOWED="*" \ No newline at end of file diff --git a/build/docker-compose.yml b/build/docker-compose.yml index fec5d1fe..d71087bd 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.7' services: grafana: - image: grafana/grafana:7.5.7 + image: grafana/grafana:9.2.4 ports: - 3000:3000 restart: unless-stopped @@ -34,7 +34,81 @@ services: - "4317:4317" # OTLP gRPC receiver - "4318:4318" # OTLP http receiver + nodelogger: + image: mojiz/nodelogger:latest + container_name: nodelogger + depends_on: + - postgres + ports: + - 2348:2345 + - ${NODELOGGER_REST_API_PORT}:${NODELOGGER_REST_API_PORT} + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + volumes: + - ./nodelogger:/go/src/app # We need it for development + environment: + LOG_LEVEL: ${NODELOGGER_LOG_LEVEL} + PROMETHEUS_URL: ${PROMETHEUS_URL} + PROMETHEUS_SYNC_INTERVAL: ${NODELOGGER_PROMETHEUS_SYNC_INTERVAL} + API_ROWS_PER_PAGE: ${NODELOGGER_API_ROWS_PER_PAGE} + REST_API_ADDRESS: ${NODELOGGER_REST_API_ADDRESS} + DEMO: ${NODELOGGER_DEMO:-true} + ORIGIN_ALLOWED: ${NODELOGGER_ORIGIN_ALLOWED} + POSTGRES_DB: ${POSTGRES_DB:-tmp} + POSTGRES_USER: ${POSTGRES_USER:-root} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} + POSTGRES_HOST: ${POSTGRES_HOST:-postgres} + healthcheck: + test: curl --fail http://nodelogger:${NODELOGGER_REST_API_PORT}/ || exit 1 + interval: 30s + retries: 10 # Will try for 5 minutes + restart: always + security_opt: + - "seccomp:unconfined" + + + postgres: + container_name: postgres + image: postgres:latest + environment: + POSTGRES_DB: ${POSTGRES_DB:-tmp} + POSTGRES_USER: ${POSTGRES_USER:-root} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} + POSTGRES_HOST: ${POSTGRES_HOST:-postgres} + volumes: + - postgres-db:/var/lib/postgresql/data + healthcheck: + test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "root" ] + timeout: 45s + interval: 10s + retries: 10 + restart: always + + + #----------------# + + pgadmin: + container_name: pgadmin + image: dpage/pgadmin4 + depends_on: + - postgres + ports: + - "${PGADMIN_PORT}:80" + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + PGADMIN_CONFIG_SERVER_MODE: 'False' + volumes: + - ./pgadmin:/root/.pgadmin + restart: unless-stopped + volumes: prometheus-data: loki-data: - grafana-data: \ No newline at end of file + grafana-data: + postgres-db: \ No newline at end of file diff --git a/tests/helpers/fund-accs/run_bridge.go b/tests/helpers/fund-accs/run_bridge.go index fd71e5ee..7d1897c7 100644 --- a/tests/helpers/fund-accs/run_bridge.go +++ b/tests/helpers/fund-accs/run_bridge.go @@ -120,7 +120,9 @@ func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { if err != nil { return err } - + + time.Sleep(5 * time.Minute) + err = nd.Stop(ctx) if err != nil { return err diff --git a/tests/helpers/fund-accs/run_light.go b/tests/helpers/fund-accs/run_light.go index 8f052981..2bbde1c1 100644 --- a/tests/helpers/fund-accs/run_light.go +++ b/tests/helpers/fund-accs/run_light.go @@ -169,6 +169,7 @@ func RunLightNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { return err } + time.Sleep(5 * time.Minute) err = nd.Stop(ctx) if err != nil { return err From 267da271fd09162ebefc37e27eb5f42e5cd35868 Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 08:18:22 +0100 Subject: [PATCH 3/7] remove nodelogger in the meantime --- build/docker-compose.yml | 146 +++++++++++++++++++-------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/build/docker-compose.yml b/build/docker-compose.yml index d71087bd..f3b65986 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -34,81 +34,81 @@ services: - "4317:4317" # OTLP gRPC receiver - "4318:4318" # OTLP http receiver - nodelogger: - image: mojiz/nodelogger:latest - container_name: nodelogger - depends_on: - - postgres - ports: - - 2348:2345 - - ${NODELOGGER_REST_API_PORT}:${NODELOGGER_REST_API_PORT} - logging: - driver: "json-file" - options: - max-size: "200k" - max-file: "10" - volumes: - - ./nodelogger:/go/src/app # We need it for development - environment: - LOG_LEVEL: ${NODELOGGER_LOG_LEVEL} - PROMETHEUS_URL: ${PROMETHEUS_URL} - PROMETHEUS_SYNC_INTERVAL: ${NODELOGGER_PROMETHEUS_SYNC_INTERVAL} - API_ROWS_PER_PAGE: ${NODELOGGER_API_ROWS_PER_PAGE} - REST_API_ADDRESS: ${NODELOGGER_REST_API_ADDRESS} - DEMO: ${NODELOGGER_DEMO:-true} - ORIGIN_ALLOWED: ${NODELOGGER_ORIGIN_ALLOWED} - POSTGRES_DB: ${POSTGRES_DB:-tmp} - POSTGRES_USER: ${POSTGRES_USER:-root} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} - POSTGRES_PORT: ${POSTGRES_PORT:-5432} - POSTGRES_HOST: ${POSTGRES_HOST:-postgres} - healthcheck: - test: curl --fail http://nodelogger:${NODELOGGER_REST_API_PORT}/ || exit 1 - interval: 30s - retries: 10 # Will try for 5 minutes - restart: always - security_opt: - - "seccomp:unconfined" - - - postgres: - container_name: postgres - image: postgres:latest - environment: - POSTGRES_DB: ${POSTGRES_DB:-tmp} - POSTGRES_USER: ${POSTGRES_USER:-root} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} - POSTGRES_PORT: ${POSTGRES_PORT:-5432} - POSTGRES_HOST: ${POSTGRES_HOST:-postgres} - volumes: - - postgres-db:/var/lib/postgresql/data - healthcheck: - test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "root" ] - timeout: 45s - interval: 10s - retries: 10 - restart: always - - - #----------------# - - pgadmin: - container_name: pgadmin - image: dpage/pgadmin4 - depends_on: - - postgres - ports: - - "${PGADMIN_PORT}:80" - environment: - PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org} - PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} - PGADMIN_CONFIG_SERVER_MODE: 'False' - volumes: - - ./pgadmin:/root/.pgadmin - restart: unless-stopped +# nodelogger: +# image: mojiz/nodelogger:latest +# container_name: nodelogger +# depends_on: +# - postgres +# ports: +# - 2348:2345 +# - ${NODELOGGER_REST_API_PORT}:${NODELOGGER_REST_API_PORT} +# logging: +# driver: "json-file" +# options: +# max-size: "200k" +# max-file: "10" +# volumes: +# - ./nodelogger:/go/src/app # We need it for development +# environment: +# LOG_LEVEL: ${NODELOGGER_LOG_LEVEL} +# PROMETHEUS_URL: ${PROMETHEUS_URL} +# PROMETHEUS_SYNC_INTERVAL: ${NODELOGGER_PROMETHEUS_SYNC_INTERVAL} +# API_ROWS_PER_PAGE: ${NODELOGGER_API_ROWS_PER_PAGE} +# REST_API_ADDRESS: ${NODELOGGER_REST_API_ADDRESS} +# DEMO: ${NODELOGGER_DEMO:-true} +# ORIGIN_ALLOWED: ${NODELOGGER_ORIGIN_ALLOWED} +# POSTGRES_DB: ${POSTGRES_DB:-tmp} +# POSTGRES_USER: ${POSTGRES_USER:-root} +# POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} +# POSTGRES_PORT: ${POSTGRES_PORT:-5432} +# POSTGRES_HOST: ${POSTGRES_HOST:-postgres} +# healthcheck: +# test: curl --fail http://nodelogger:${NODELOGGER_REST_API_PORT}/ || exit 1 +# interval: 30s +# retries: 10 # Will try for 5 minutes +# restart: always +# security_opt: +# - "seccomp:unconfined" +# +# +# postgres: +# container_name: postgres +# image: postgres:latest +# environment: +# POSTGRES_DB: ${POSTGRES_DB:-tmp} +# POSTGRES_USER: ${POSTGRES_USER:-root} +# POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} +# POSTGRES_PORT: ${POSTGRES_PORT:-5432} +# POSTGRES_HOST: ${POSTGRES_HOST:-postgres} +# volumes: +# - postgres-db:/var/lib/postgresql/data +# healthcheck: +# test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "root" ] +# timeout: 45s +# interval: 10s +# retries: 10 +# restart: always +# +# +# #----------------# +# +# pgadmin: +# container_name: pgadmin +# image: dpage/pgadmin4 +# depends_on: +# - postgres +# ports: +# - "${PGADMIN_PORT}:80" +# environment: +# PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org} +# PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} +# PGADMIN_CONFIG_SERVER_MODE: 'False' +# volumes: +# - ./pgadmin:/root/.pgadmin +# restart: unless-stopped volumes: prometheus-data: loki-data: grafana-data: - postgres-db: \ No newline at end of file +# postgres-db: \ No newline at end of file From 7b264d34afce75ee675955e681f20de57352984a Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 08:19:37 +0100 Subject: [PATCH 4/7] fix loglevel --- build/otel-collector/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/otel-collector/config.yml b/build/otel-collector/config.yml index 9eca6b7d..4a9e70b8 100644 --- a/build/otel-collector/config.yml +++ b/build/otel-collector/config.yml @@ -11,7 +11,7 @@ receivers: exporters: logging: - logLevel: debug + loglevel: debug prometheus: endpoint: "otel-collector:8889" namespace: "testground" From 4410c0272fb1192022f63f57413973cdaed9ad56 Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 08:21:10 +0100 Subject: [PATCH 5/7] fix port prom --- build/grafana/config.yml | 2 +- build/otel-collector/config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/grafana/config.yml b/build/grafana/config.yml index 2d8ca1db..30b84c37 100644 --- a/build/grafana/config.yml +++ b/build/grafana/config.yml @@ -2,5 +2,5 @@ datasources: - name: Prometheus access: proxy type: prometheus - url: http://0.0.0.0:9090 + url: http://0.0.0.0:9000 isDefault: true \ No newline at end of file diff --git a/build/otel-collector/config.yml b/build/otel-collector/config.yml index 4a9e70b8..28bbfa62 100644 --- a/build/otel-collector/config.yml +++ b/build/otel-collector/config.yml @@ -16,7 +16,7 @@ exporters: endpoint: "otel-collector:8889" namespace: "testground" send_timestamps: true - metric_expiration: 30m + metric_expiration: 120m processors: batch: From 7534bcb908b43811f7a865731718c47a5cb044cc Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 16:54:04 +0100 Subject: [PATCH 6/7] chore: make otel-collector testing --- compositions/cluster-k8s/pfd-1100.toml | 2 +- compositions/cluster-k8s/pfd-400.toml | 16 +++---- tests/helpers/common/bridge.go | 5 ++- tests/helpers/common/validator.go | 2 +- tests/helpers/fund-accs/run_bridge.go | 60 +++++++++++++------------- tests/helpers/fund-accs/run_light.go | 2 +- 6 files changed, 44 insertions(+), 43 deletions(-) diff --git a/compositions/cluster-k8s/pfd-1100.toml b/compositions/cluster-k8s/pfd-1100.toml index 301290a9..34fbe170 100644 --- a/compositions/cluster-k8s/pfd-1100.toml +++ b/compositions/cluster-k8s/pfd-1100.toml @@ -20,7 +20,7 @@ validator = "100" bridge = "100" light = "1000" - submit-times = "10" + submit-times = "20" [[groups]] id = "seeds" diff --git a/compositions/cluster-k8s/pfd-400.toml b/compositions/cluster-k8s/pfd-400.toml index 12394c5d..420f4db7 100644 --- a/compositions/cluster-k8s/pfd-400.toml +++ b/compositions/cluster-k8s/pfd-400.toml @@ -1,5 +1,5 @@ [metadata] - name = "pay-for-data-100-100-1000-set" + name = "pay-for-data-100-100-100-set" author = "Bidon15" [global] @@ -13,14 +13,14 @@ artifact = "" [global.run.test_params] execution-time = "25" - msg-size = "4800" + msg-size = "50000" namespace-id = "random" persistent-peers = "10" seed = "5" validator = "100" bridge = "100" light = "100" - submit-times = "3" + submit-times = "20" [[groups]] id = "seeds" @@ -70,8 +70,8 @@ id = "bridges" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "3000m" + memory = "6Gi" + cpu = "5000m" [groups.instances] count = 100 percentage = 0.0 @@ -85,7 +85,7 @@ artifact = "" [groups.run.test_params] bandwidth = "320Mib" - block-height = "20" + block-height = "37" latency = "0" role = "bridge" @@ -93,8 +93,8 @@ id = "lights" builder = "docker:generic" [groups.resources] - memory = "300Mi" - cpu = "200m" + memory = "500Mi" + cpu = "500m" [groups.instances] count = 100 percentage = 0.0 diff --git a/tests/helpers/common/bridge.go b/tests/helpers/common/bridge.go index 9f5ec8d2..39e9bfaa 100644 --- a/tests/helpers/common/bridge.go +++ b/tests/helpers/common/bridge.go @@ -3,6 +3,7 @@ package common import ( "context" "fmt" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -14,7 +15,6 @@ import ( "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" "github.com/testground/sdk-go/sync" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" ) func BuildBridge(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) (*nodebuilder.Node, error) { @@ -57,7 +57,8 @@ func BuildBridge(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitC } nd, err := nodekit.NewNode(ndhome, node.Bridge, - runenv.StringParam("p2p-network"), cfg, nodebuilder.WithMetrics( + runenv.StringParam("p2p-network"), cfg, + nodebuilder.WithMetrics( optlOpts, node.Bridge, )) diff --git a/tests/helpers/common/validator.go b/tests/helpers/common/validator.go index 4400cf0f..610a0b19 100644 --- a/tests/helpers/common/validator.go +++ b/tests/helpers/common/validator.go @@ -268,7 +268,7 @@ func changeConfig(path, mempool string) error { "timeout_propose": "10s", "timeout_prevote": "1s", "timeout_precommit": "1s", - "timeout_commit": "15s", + "timeout_commit": "10s", }, "rpc": { "max_subscriptions_per_client": 150, diff --git a/tests/helpers/fund-accs/run_bridge.go b/tests/helpers/fund-accs/run_bridge.go index 7d1897c7..b0cba205 100644 --- a/tests/helpers/fund-accs/run_bridge.go +++ b/tests/helpers/fund-accs/run_bridge.go @@ -92,36 +92,36 @@ func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { runenv.RecordFailure(fmt.Errorf("bridge node is still syncing the past")) } - bal, err := nd.StateServ.Balance(ctx) - if err != nil { - return err - } - if bal.IsZero() { - return fmt.Errorf("bridge has no money in the bank") - } - - runenv.RecordMessage("bridge -> %d has this %s balance", initCtx.GroupSeq, bal.String()) - - nid := common.GenerateNamespaceID(runenv.StringParam("namespace-id")) - data := common.GetRandomMessageBySize(runenv.IntParam("msg-size")) - - for i := 0; i < runenv.IntParam("submit-times"); i++ { - err = common.SubmitData(ctx, runenv, nd, nid, data) - if err != nil { - return err - } - - if runenv.TestCase == "get-shares-by-namespace" && common.VerifyDataInNamespace(ctx, nd, nid, data) != nil { - return fmt.Errorf("no expected data found in the namespace ID") - } - } - - err = common.CheckBalanceDeduction(ctx, nd, bal) - if err != nil { - return err - } - - time.Sleep(5 * time.Minute) + //bal, err := nd.StateServ.Balance(ctx) + //if err != nil { + // return err + //} + //if bal.IsZero() { + // return fmt.Errorf("bridge has no money in the bank") + //} + // + //runenv.RecordMessage("bridge -> %d has this %s balance", initCtx.GroupSeq, bal.String()) + // + //nid := common.GenerateNamespaceID(runenv.StringParam("namespace-id")) + //data := common.GetRandomMessageBySize(runenv.IntParam("msg-size")) + // + //for i := 0; i < runenv.IntParam("submit-times"); i++ { + // err = common.SubmitData(ctx, runenv, nd, nid, data) + // if err != nil { + // return err + // } + // + // if runenv.TestCase == "get-shares-by-namespace" && common.VerifyDataInNamespace(ctx, nd, nid, data) != nil { + // return fmt.Errorf("no expected data found in the namespace ID") + // } + //} + // + //err = common.CheckBalanceDeduction(ctx, nd, bal) + //if err != nil { + // return err + //} + + //time.Sleep(5 * time.Minute) err = nd.Stop(ctx) if err != nil { diff --git a/tests/helpers/fund-accs/run_light.go b/tests/helpers/fund-accs/run_light.go index 2bbde1c1..efb09969 100644 --- a/tests/helpers/fund-accs/run_light.go +++ b/tests/helpers/fund-accs/run_light.go @@ -169,7 +169,7 @@ func RunLightNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { return err } - time.Sleep(5 * time.Minute) + //time.Sleep(5 * time.Minute) err = nd.Stop(ctx) if err != nil { return err From 377d1b0b1c6ab938a2de35e9948b2c5bba44d34b Mon Sep 17 00:00:00 2001 From: Nhu Viet Nguyen Date: Wed, 15 Feb 2023 21:09:30 +0100 Subject: [PATCH 7/7] fix port prom to k8s --- compositions/cluster-k8s/pfd-1100.toml | 12 ++++++------ manifest.toml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compositions/cluster-k8s/pfd-1100.toml b/compositions/cluster-k8s/pfd-1100.toml index 34fbe170..7ba4558a 100644 --- a/compositions/cluster-k8s/pfd-1100.toml +++ b/compositions/cluster-k8s/pfd-1100.toml @@ -48,8 +48,8 @@ id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "3500m" + memory = "6Gi" + cpu = "6000m" [groups.instances] count = 100 percentage = 0.0 @@ -70,8 +70,8 @@ id = "bridges" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "3000m" + memory = "8Gi" + cpu = "4000m" [groups.instances] count = 100 percentage = 0.0 @@ -93,8 +93,8 @@ id = "lights" builder = "docker:generic" [groups.resources] - memory = "300Mi" - cpu = "200m" + memory = "600Mi" + cpu = "600m" [groups.instances] count = 1000 percentage = 0.0 diff --git a/manifest.toml b/manifest.toml index 30b29444..501ffa82 100644 --- a/manifest.toml +++ b/manifest.toml @@ -125,7 +125,7 @@ instances = { min = 4, max = 3000, default = 12 } block-height = { type = "int" } role = { type = "string" } p2p-network = { type = "string", default = "private" } - otel-collector-address = { type = "string", default = "46.101.22.123:4318" } + otel-collector-address = { type = "string", default = "otlp.celestia.tools:4318" } [[testcases]] name = "get-shares-by-namespace" @@ -182,7 +182,7 @@ instances = { min = 30, max = 1002, default = 30 } block-height = { type = "int" } role = { type = "string" } p2p-network = { type = "string", default = "private" } - otel-collector-address = { type = "string", default = "46.101.22.123:4318" } + otel-collector-address = { type = "string", default = "otlp.celestia.tools:4318" } [[testcases]] name = "flood-arabica"