feat(#9428): collect new size metrics from CHT monitoring API (#114)

medic · Sep 18, 2024 · b043b5d · b043b5d
1 parent ed30434
commit b043b5d
Show file tree

Hide file tree

Showing 7 changed files with 302 additions and 7 deletions.
diff --git a/.env.example b/.env.example
@@ -24,6 +24,9 @@ SQL_EXPORTER_IP=127.0.0.1
 # 9399 is default from upstream project
 SQL_EXPORTER_PORT=9399
 
+# Use to customize log retention for docker containers
+#LOG_MAX_SIZE=50m
+#LOG_MAX_FILES=20
 
 # For deveopment use only!!
 # If you want to expose your prometheus container on your local dev host, uncomment this and set to

diff --git a/development/README.md b/development/README.md
@@ -57,7 +57,7 @@ Tips:
 
 ### Adding a new metric from CHT /monitoring API
 
-Metrics loaded from the CHT `/monitoring` endpoint are configured via the json-exporter's [`config.yml`](../exporters/json/config/cht.yml) file. The `modules.default.metrics` section contains the configuration for mapping the JSON response from the CHT `/monitoring` endpoint to Prometheus metrics. New values added to this JSON can be included in Prometheus by adding additional mapping here. See the [json_exporter project](https://github.com/prometheus-community/json_exporter) on GitHub for more information.
+Metrics loaded from the CHT `/monitoring` endpoint are configured via the json-exporter's [`config/cht.yml`](../exporters/json/config/cht.yml) file. The `modules.default.metrics` section contains the configuration for mapping the JSON response from the CHT `/monitoring` endpoint to Prometheus metrics. New values added to this JSON can be included in Prometheus by adding additional mapping here. See the [json_exporter project](https://github.com/prometheus-community/json_exporter) on GitHub for more information.
 
 ### Adding a new metric from a Couch2pg Postgres DB
 

diff --git a/development/fake-cht/initial-response.json b/development/fake-cht/initial-response.json
@@ -10,28 +10,115 @@
       "update_sequence": 0,
       "doc_count": 0,
       "doc_del_count": 0,
-      "fragmentation": 0
+      "fragmentation": 0,
+      "sizes": {
+        "file": 0,
+        "active": 0
+      },
+      "view_indexes": [
+        {
+          "name": "medic",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        },
+        {
+          "name": "medic-admin",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        },
+        {
+          "name": "medic-client",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        },
+        {
+          "name": "medic-conflicts",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        },
+        {
+          "name": "medic-scripts",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        },
+        {
+          "name": "medic-sms",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        }
+      ]
     },
     "sentinel": {
       "name": "medic-sentinel",
       "update_sequence": 0,
       "doc_count": 0,
       "doc_del_count": 0,
-      "fragmentation": 0
+      "fragmentation": 0,
+      "sizes": {
+        "file": 0,
+        "active": 0
+      },
+      "view_indexes": [
+        {
+          "name": "sentinel",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        }
+      ]
     },
     "usersmeta": {
       "name": "medic-users-meta",
       "update_sequence": 0,
       "doc_count": 0,
       "doc_del_count": 0,
-      "fragmentation": 0
+      "fragmentation": 0,
+      "sizes": {
+        "file": 0,
+        "active": 0
+      },
+      "view_indexes": [
+        {
+          "name": "users-meta",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        }
+      ]
     },
     "users": {
       "name": "_users",
       "update_sequence": 0,
       "doc_count": 0,
       "doc_del_count": 0,
-      "fragmentation": 0
+      "fragmentation": 0,
+      "sizes": {
+        "file": 0,
+        "active": 0
+      },
+      "view_indexes": [
+        {
+          "name": "users",
+          "sizes": {
+            "file": 0,
+            "active": 0
+          }
+        }
+      ]
     }
   },
   "date": {

diff --git a/development/fake-cht/src/index.js b/development/fake-cht/src/index.js
@@ -4,6 +4,20 @@ const initialResponse = require('../initial-response.json');
 const prometheusMiddleware = require('prometheus-api-metrics');
 const { updatePostgres } = require('./postgres');
 
+const VIEW_INDEXES_BY_DB = {
+  ['medic']: [
+    'medic',
+    'medic-admin',
+    'medic-client',
+    'medic-conflicts',
+    'medic-scripts',
+    'medic-sms',
+  ],
+  ['medic-sentinel']: ['sentinel'],
+  ['medic-users-meta']: ['users-meta'],
+  _users: ['users'],
+};
+
 const getRandomInt = (min, max) => Math.floor(Math.random() * (max - min)) + min;
 
 const randomGauge = (min, max, current, maxChange) => {
@@ -18,12 +32,25 @@ const randomCounter = (current, changeFactor) => {
 
 const getVersion = () => ({ app: '4.1.1', node: 'v16.17.1', couchdb: '2.3.1' });
 
-const getCouchDb = ({ name, update_sequence, doc_count, doc_del_count, fragmentation }) => ({
+const getSizes = ({ active, file }) => ({
+  active: randomGauge(0, 100000000, active, 10000),
+  file: randomGauge(0, 100000000, file, 10000),
+});
+
+const getViewIndex = ({ name, sizes }) => ({
+  name,
+  sizes: getSizes(sizes),
+});
+
+const getCouchDb = ({ name, update_sequence, doc_count, doc_del_count, fragmentation, sizes, view_indexes }) => ({
   name,
   update_sequence: randomCounter(update_sequence, 100),
   doc_count: randomCounter(doc_count, 100),
   doc_del_count: randomCounter(doc_del_count, 1),
   fragmentation: randomGauge(1, 10, fragmentation, 1) + Math.random(),
+  sizes: getSizes(sizes),
+  view_indexes: VIEW_INDEXES_BY_DB[name]
+    .map(viewIndexName => getViewIndex(view_indexes.find(viewIndex => viewIndex.name === viewIndexName))),
 });
 
 const getAllCouchDbs = ({ medic, sentinel, usersmeta, users }) => ({

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -29,6 +29,11 @@ services:
       - ./cht-instances.yml:/etc/prometheus/cht-instances.yml:ro
       - ./exporters/json/config/scrape_config.yml:/etc/prometheus/scrape_configs/cht-json.yml:ro
       - prometheus-data:/prometheus
+    logging:
+      driver: "local"
+      options:
+        max-size: "${LOG_MAX_SIZE:-50m}"
+        max-file: "${LOG_MAX_FILES:-20}"
     networks:
       - cht-watchdog-net
     restart: always
@@ -46,6 +51,11 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-password}
       - GF_USERS_ALLOW_SIGN_UP=false
       - GF_INSTALL_PLUGINS=${GRAFANA_PLUGINS:-grafana-discourse-datasource}
+    logging:
+      driver: "local"
+      options:
+        max-size: "${LOG_MAX_SIZE:-50m}"
+        max-file: "${LOG_MAX_FILES:-20}"
     networks:
       - cht-watchdog-net
     restart: always
@@ -54,6 +64,11 @@ services:
     image: prometheuscommunity/json-exporter:${JSON_EXPORTER_VERSION:-v0.6.0}
     volumes:
       - ./exporters/json/config/cht.yml:/config.yml:ro
+    logging:
+      driver: "local"
+      options:
+        max-size: "${LOG_MAX_SIZE:-50m}"
+        max-file: "${LOG_MAX_FILES:-20}"
     networks:
       - cht-watchdog-net
     restart: always