From 72a48ba8c5481b642386683509bb08b672c0eb60 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 11 Dec 2023 18:29:23 -0800 Subject: [PATCH 1/3] Add metrics for OS and OSD processes Signed-off-by: Sayali Gaikawad --- lib/cloudwatch/metrics-section.ts | 31 ++++++++++++++----- lib/infra/infra-stack.ts | 49 +++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/lib/cloudwatch/metrics-section.ts b/lib/cloudwatch/metrics-section.ts index 36f1931c6af..1bcbb35c81d 100644 --- a/lib/cloudwatch/metrics-section.ts +++ b/lib/cloudwatch/metrics-section.ts @@ -5,18 +5,35 @@ The OpenSearch Contributors require contributions made to this file be licensed under the Apache-2.0 license or a compatible open source license. */ +import { Unit } from 'aws-cdk-lib/aws-cloudwatch'; + +type MeasurementDefinition = string | { name: string, rename?: string, unit?: Unit } + interface MetricDefinition { - measurement: string[]; -} + resources?: string[], + measurement: MeasurementDefinition[], + // eslint-disable-next-line camelcase + metrics_collection_interval?: number, + } + +export interface ProcstatMetricDefinition { + pattern?: string; + // eslint-disable-next-line camelcase + append_dimensions?: string[]; + measurement: string[]; // procstat doesn't support the common measurement standard for rename/unit + // eslint-disable-next-line camelcase + metrics_collection_interval: number; + } interface EditableCloudwatchMetricsSection { // eslint-disable-next-line camelcase metrics_collected: { - cpu: MetricDefinition, - disk: MetricDefinition, - diskio: MetricDefinition, - mem: MetricDefinition, - net: MetricDefinition, + procstat?: ProcstatMetricDefinition[], + cpu?: MetricDefinition, + disk?: MetricDefinition, + diskio?: MetricDefinition, + mem?: MetricDefinition, + net?: MetricDefinition, }; } diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 97f36be37e0..8e7b1eef2f5 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -11,6 +11,7 @@ import { import { AutoScalingGroup, BlockDeviceVolume, EbsDeviceVolumeType, Signals, } from 'aws-cdk-lib/aws-autoscaling'; +import { Unit } from 'aws-cdk-lib/aws-cloudwatch'; import { AmazonLinuxCpuType, AmazonLinuxGeneration, @@ -38,6 +39,7 @@ import { readFileSync } from 'fs'; import { dump, load } from 'js-yaml'; import { join } from 'path'; import { CloudwatchAgent } from '../cloudwatch/cloudwatch-agent'; +import { ProcstatMetricDefinition } from '../cloudwatch/metrics-section'; import { nodeConfig } from '../opensearch-config/node-config'; import { RemoteStoreResources } from './remote-store-resources'; @@ -375,6 +377,33 @@ export class InfraStack extends Stack { private static getCfnInitElement(scope: Stack, logGroup: LogGroup, props: infraProps, nodeType?: string): InitElement[] { const configFileDir = join(__dirname, '../opensearch-config'); let opensearchConfig: string; + const procstatConfig: ProcstatMetricDefinition[] = [{ + pattern: '-Dopensearch', + measurement: [ + 'cpu_usage', + 'cpu_time_system', + 'cpu_time_user', + 'read_bytes', + 'write_bytes', + 'pid_count', + ], + metrics_collection_interval: 10, + }, + ]; + if (props.dashboardsUrl !== 'undefined') { + procstatConfig.push({ + pattern: 'opensearch-dashboards', + measurement: [ + 'cpu_usage', + 'cpu_time_system', + 'cpu_time_user', + 'read_bytes', + 'write_bytes', + 'pid_count', + ], + metrics_collection_interval: 15, + }); + } const cfnInitConfig: InitElement[] = [ InitPackage.yum('amazon-cloudwatch-agent'), @@ -388,6 +417,7 @@ export class InfraStack extends Stack { }, metrics: { metrics_collected: { + procstat: procstatConfig, cpu: { measurement: [ // eslint-disable-next-line max-len @@ -396,7 +426,13 @@ export class InfraStack extends Stack { }, disk: { measurement: [ - 'free', 'total', 'used', 'used_percent', 'inodes_free', 'inodes_used', 'inodes_total', + { name: 'free', unit: Unit.PERCENT }, + { name: 'total', unit: Unit.PERCENT }, + { name: 'used', unit: Unit.PERCENT }, + { name: 'used_percent', unit: Unit.PERCENT }, + { name: 'inodes_free', unit: Unit.PERCENT }, + { name: 'inodes_used', unit: Unit.PERCENT }, + { name: 'inodes_total', unit: Unit.PERCENT }, ], }, diskio: { @@ -406,7 +442,16 @@ export class InfraStack extends Stack { }, mem: { measurement: [ - 'active', 'available', 'available_percent', 'buffered', 'cached', 'free', 'inactive', 'total', 'used', 'used_percent', + { name: 'active', unit: Unit.PERCENT }, + { name: 'available', unit: Unit.PERCENT }, + { name: 'available_percent', unit: Unit.PERCENT }, + { name: 'buffered', unit: Unit.PERCENT }, + { name: 'cached', unit: Unit.PERCENT }, + { name: 'free', unit: Unit.PERCENT }, + { name: 'inactive', unit: Unit.PERCENT }, + { name: 'total', unit: Unit.PERCENT }, + { name: 'used', unit: Unit.PERCENT }, + { name: 'used_percent', unit: Unit.PERCENT }, ], }, net: { From d9b2f1001d40e37ae8487c37d4d6b8fa8403de89 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 11 Dec 2023 18:31:44 -0800 Subject: [PATCH 2/3] typo Signed-off-by: Sayali Gaikawad --- lib/cloudwatch/metrics-section.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cloudwatch/metrics-section.ts b/lib/cloudwatch/metrics-section.ts index 1bcbb35c81d..ca4f74d1fe8 100644 --- a/lib/cloudwatch/metrics-section.ts +++ b/lib/cloudwatch/metrics-section.ts @@ -20,7 +20,7 @@ export interface ProcstatMetricDefinition { pattern?: string; // eslint-disable-next-line camelcase append_dimensions?: string[]; - measurement: string[]; // procstat doesn't support the common measurement standard for rename/unit + measurement: string[]; // procstat does not support the common measurement standard for rename/unit // eslint-disable-next-line camelcase metrics_collection_interval: number; } From e96237b849dda213af3dc8caa553a345623dd906 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Wed, 13 Dec 2023 10:36:23 -0800 Subject: [PATCH 3/3] Keep pid Signed-off-by: Sayali Gaikawad --- lib/infra/infra-stack.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 8e7b1eef2f5..f0831ab9847 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -380,11 +380,6 @@ export class InfraStack extends Stack { const procstatConfig: ProcstatMetricDefinition[] = [{ pattern: '-Dopensearch', measurement: [ - 'cpu_usage', - 'cpu_time_system', - 'cpu_time_user', - 'read_bytes', - 'write_bytes', 'pid_count', ], metrics_collection_interval: 10, @@ -394,11 +389,6 @@ export class InfraStack extends Stack { procstatConfig.push({ pattern: 'opensearch-dashboards', measurement: [ - 'cpu_usage', - 'cpu_time_system', - 'cpu_time_user', - 'read_bytes', - 'write_bytes', 'pid_count', ], metrics_collection_interval: 15,