Add metrics for OS and OSD processes (#83)

Signed-off-by: Sayali Gaikawad <[email protected]>
opensearch-project · Dec 13, 2023 · 5ea6747 · 5ea6747
1 parent 68679fc
commit 5ea6747
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 9 deletions.
diff --git a/lib/cloudwatch/metrics-section.ts b/lib/cloudwatch/metrics-section.ts
@@ -5,18 +5,35 @@ The OpenSearch Contributors require contributions made to
 this file be licensed under the Apache-2.0 license or a
 compatible open source license. */
 
+import { Unit } from 'aws-cdk-lib/aws-cloudwatch';
+
+type MeasurementDefinition = string | { name: string, rename?: string, unit?: Unit }
+
 interface MetricDefinition {
-    measurement: string[];
-}
+    resources?: string[],
+    measurement: MeasurementDefinition[],
+    // eslint-disable-next-line camelcase
+    metrics_collection_interval?: number,
+  }
+
+export interface ProcstatMetricDefinition {
+    pattern?: string;
+    // eslint-disable-next-line camelcase
+    append_dimensions?: string[];
+    measurement: string[]; // procstat does not support the common measurement standard for rename/unit
+    // eslint-disable-next-line camelcase
+    metrics_collection_interval: number;
+  }
 
 interface EditableCloudwatchMetricsSection {
     // eslint-disable-next-line camelcase
     metrics_collected: {
-        cpu: MetricDefinition,
-        disk: MetricDefinition,
-        diskio: MetricDefinition,
-        mem: MetricDefinition,
-        net: MetricDefinition,
+        procstat?: ProcstatMetricDefinition[],
+        cpu?: MetricDefinition,
+        disk?: MetricDefinition,
+        diskio?: MetricDefinition,
+        mem?: MetricDefinition,
+        net?: MetricDefinition,
     };
 }
 

diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts
@@ -11,6 +11,7 @@ import {
 import {
   AutoScalingGroup, BlockDeviceVolume, EbsDeviceVolumeType, Signals,
 } from 'aws-cdk-lib/aws-autoscaling';
+import { Unit } from 'aws-cdk-lib/aws-cloudwatch';
 import {
   AmazonLinuxCpuType,
   AmazonLinuxGeneration,
@@ -38,6 +39,7 @@ import { readFileSync } from 'fs';
 import { dump, load } from 'js-yaml';
 import { join } from 'path';
 import { CloudwatchAgent } from '../cloudwatch/cloudwatch-agent';
+import { ProcstatMetricDefinition } from '../cloudwatch/metrics-section';
 import { nodeConfig } from '../opensearch-config/node-config';
 import { RemoteStoreResources } from './remote-store-resources';
 
@@ -375,6 +377,23 @@ export class InfraStack extends Stack {
   private static getCfnInitElement(scope: Stack, logGroup: LogGroup, props: infraProps, nodeType?: string): InitElement[] {
     const configFileDir = join(__dirname, '../opensearch-config');
     let opensearchConfig: string;
+    const procstatConfig: ProcstatMetricDefinition[] = [{
+      pattern: '-Dopensearch',
+      measurement: [
+        'pid_count',
+      ],
+      metrics_collection_interval: 10,
+    },
+    ];
+    if (props.dashboardsUrl !== 'undefined') {
+      procstatConfig.push({
+        pattern: 'opensearch-dashboards',
+        measurement: [
+          'pid_count',
+        ],
+        metrics_collection_interval: 15,
+      });
+    }
 
     const cfnInitConfig: InitElement[] = [
       InitPackage.yum('amazon-cloudwatch-agent'),
@@ -388,6 +407,7 @@ export class InfraStack extends Stack {
           },
           metrics: {
             metrics_collected: {
+              procstat: procstatConfig,
               cpu: {
                 measurement: [
                   // eslint-disable-next-line max-len
@@ -396,7 +416,13 @@ export class InfraStack extends Stack {
               },
               disk: {
                 measurement: [
-                  'free', 'total', 'used', 'used_percent', 'inodes_free', 'inodes_used', 'inodes_total',
+                  { name: 'free', unit: Unit.PERCENT },
+                  { name: 'total', unit: Unit.PERCENT },
+                  { name: 'used', unit: Unit.PERCENT },
+                  { name: 'used_percent', unit: Unit.PERCENT },
+                  { name: 'inodes_free', unit: Unit.PERCENT },
+                  { name: 'inodes_used', unit: Unit.PERCENT },
+                  { name: 'inodes_total', unit: Unit.PERCENT },
                 ],
               },
               diskio: {
@@ -406,7 +432,16 @@ export class InfraStack extends Stack {
               },
               mem: {
                 measurement: [
-                  'active', 'available', 'available_percent', 'buffered', 'cached', 'free', 'inactive', 'total', 'used', 'used_percent',
+                  { name: 'active', unit: Unit.PERCENT },
+                  { name: 'available', unit: Unit.PERCENT },
+                  { name: 'available_percent', unit: Unit.PERCENT },
+                  { name: 'buffered', unit: Unit.PERCENT },
+                  { name: 'cached', unit: Unit.PERCENT },
+                  { name: 'free', unit: Unit.PERCENT },
+                  { name: 'inactive', unit: Unit.PERCENT },
+                  { name: 'total', unit: Unit.PERCENT },
+                  { name: 'used', unit: Unit.PERCENT },
+                  { name: 'used_percent', unit: Unit.PERCENT },
                 ],
               },
               net: {