[Backport 1.x] Add support to add additional configuration to opensearch-dashboards and fix readme

README.md

@@ -55,6 +55,7 @@ In order to deploy both the stacks the user needs to provide a set of required a
 | mlInstanceType         | Optional    | string  | EC2 instance type for ml node. Defaults to r5.xlarge. See options in `lib/opensearch-config/node-config.ts` for available options. E.g., `-c mlInstanceType=m5.xlarge`                                                                                                                           |
 | jvmSysProps            | Optional    | string  | A comma-separated list of key=value pairs that will be added to `jvm.options` as JVM system properties.                                                                                                                                                                                          |
 | additionalConfig       | Optional    | string  | Additional opensearch.yml config parameters passed as JSON. e.g., `--context additionalConfig='{"plugins.security.nodes_dn": ["CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE", "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"], "plugins.security.nodes_dn_dynamic_config_enabled": false}'` |
+| additionalOsdConfig       | Optional    | string  | Additional opensearch_dashboards.yml config parameters passed as JSON. e.g., `additionalOsdConfig='{"data.search.usageTelemetry.enabled": "true"}'` |
 | suffix                 | Optional    | string  | An optional string identifier to be concatenated with infra stack name.                                                                                                                                                                                                                          |
 | region                 | Optional    | string  | User provided aws region                                                                                                                                                                                                                                                                         |
 | account                | Optional    | string  | User provided aws account                                                                                                                                                                                                                                                                        |
@@ -86,7 +87,7 @@ cdk bootstrap aws://<aws-account-number>/<aws-region> --context securityDisabled
 * Now you are ready to synthesize the CloudFormation templates:
 
 ```
-cdk synth "* " --context securityDisabled=true \
+cdk synth "*" --context securityDisabled=true \
 --context minDistribution=false --context distributionUrl='https://artifacts.opensearch.org/releases/bundle/opensearch/2.3.0/opensearch-2.3.0-linux-x64.tar.gz' \
 --context cpuArch='x64' --context singleNodeCluster=false --context dataNodeCount=3 \
 --context dashboardsUrl='https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.3.0/opensearch-dashboards-2.3.0-linux-x64.tar.gz' \

lib/infra/infra-stack.ts

@@ -5,63 +5,65 @@
 this file be licensed under the Apache-2.0 license or a
 compatible open source license. */
 
+import {
+  CfnOutput, RemovalPolicy, Stack, StackProps, Tags,
+} from 'aws-cdk-lib';
+import { AutoScalingGroup, BlockDeviceVolume, Signals } from 'aws-cdk-lib/aws-autoscaling';
 import {
   AmazonLinuxCpuType,
   AmazonLinuxGeneration,
   CloudFormationInit,
+  ISecurityGroup,
+  IVpc,
   InitCommand,
   InitElement,
   InitPackage,
   Instance,
   InstanceClass,
   InstanceSize,
   InstanceType,
-  ISecurityGroup,
-  IVpc,
   MachineImage,
   SubnetType,
 } from 'aws-cdk-lib/aws-ec2';
-import { AutoScalingGroup, BlockDeviceVolume, Signals } from 'aws-cdk-lib/aws-autoscaling';
+import { NetworkListener, NetworkLoadBalancer, Protocol } from 'aws-cdk-lib/aws-elasticloadbalancingv2';
+import { InstanceTarget } from 'aws-cdk-lib/aws-elasticloadbalancingv2-targets';
 import {
-  ManagedPolicy, Role, IRole, ServicePrincipal,
+  ManagedPolicy, Role,
+  ServicePrincipal,
 } from 'aws-cdk-lib/aws-iam';
 import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs';
-import {
-  CfnOutput, RemovalPolicy, Stack, StackProps, Tags,
-} from 'aws-cdk-lib';
-import { NetworkListener, NetworkLoadBalancer, Protocol } from 'aws-cdk-lib/aws-elasticloadbalancingv2';
-import { join } from 'path';
 import { readFileSync } from 'fs';
 import { dump, load } from 'js-yaml';
-import { InstanceTarget } from 'aws-cdk-lib/aws-elasticloadbalancingv2-targets';
+import { join } from 'path';
 import { CloudwatchAgent } from '../cloudwatch/cloudwatch-agent';
 import { nodeConfig } from '../opensearch-config/node-config';
 
 export interface infraProps extends StackProps {
-    readonly vpc: IVpc,
-    readonly securityGroup: ISecurityGroup,
-    readonly opensearchVersion: string,
-    readonly cpuArch: string,
-    readonly cpuType: AmazonLinuxCpuType,
-    readonly securityDisabled: boolean,
-    readonly minDistribution: boolean,
-    readonly distributionUrl: string,
-    readonly dashboardsUrl: string,
-    readonly singleNodeCluster: boolean,
-    readonly managerNodeCount: number,
-    readonly dataNodeCount: number,
-    readonly ingestNodeCount: number,
-    readonly clientNodeCount: number,
-    readonly mlNodeCount: number,
-    readonly dataNodeStorage: number,
-    readonly mlNodeStorage: number,
-    readonly jvmSysPropsString?: string,
-    readonly additionalConfig?: string,
-    readonly dataEc2InstanceType: InstanceType,
-    readonly mlEc2InstanceType: InstanceType,
-    readonly use50PercentHeap: boolean,
-    readonly isInternal: boolean,
-    readonly customRoleArn: string
+  readonly vpc: IVpc,
+  readonly securityGroup: ISecurityGroup,
+  readonly opensearchVersion: string,
+  readonly cpuArch: string,
+  readonly cpuType: AmazonLinuxCpuType,
+  readonly securityDisabled: boolean,
+  readonly minDistribution: boolean,
+  readonly distributionUrl: string,
+  readonly dashboardsUrl: string,
+  readonly singleNodeCluster: boolean,
+  readonly managerNodeCount: number,
+  readonly dataNodeCount: number,
+  readonly ingestNodeCount: number,
+  readonly clientNodeCount: number,
+  readonly mlNodeCount: number,
+  readonly dataNodeStorage: number,
+  readonly mlNodeStorage: number,
+  readonly jvmSysPropsString?: string,
+  readonly additionalConfig?: string,
+  readonly additionalOsdConfig?: string,
+  readonly dataEc2InstanceType: InstanceType,
+  readonly mlEc2InstanceType: InstanceType,
+  readonly use50PercentHeap: boolean,
+  readonly isInternal: boolean,
+  readonly customRoleArn: string
 }
 
 export class InfraStack extends Stack {
@@ -127,7 +129,7 @@
     }
 
     if (props.singleNodeCluster) {
       console.log('Single node value is true, creating single node configurations');
       singleNodeInstance = new Instance(this, 'single-node-instance', {
         vpc: props.vpc,
         instanceType: singleNodeInstanceType,
@@ -355,7 +357,7 @@
     const configFileDir = join(__dirname, '../opensearch-config');
     let opensearchConfig: string;
 
-    const cfnInitConfig : InitElement[] = [
+    const cfnInitConfig: InitElement[] = [
       InitPackage.yum('amazon-cloudwatch-agent'),
       CloudwatchAgent.asInitFile('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
         {
@@ -417,7 +419,7 @@
       InitCommand.shellCommand('set -ex;/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s'),
       InitCommand.shellCommand('set -ex; sudo echo "vm.max_map_count=262144" >> /etc/sysctl.conf;sudo sysctl -p'),
       InitCommand.shellCommand(`set -ex;mkdir opensearch; curl -L ${props.distributionUrl} -o opensearch.tar.gz;`
-                + 'tar zxf opensearch.tar.gz -C opensearch --strip-components=1; chown -R ec2-user:ec2-user opensearch;', {
+        + 'tar zxf opensearch.tar.gz -C opensearch --strip-components=1; chown -R ec2-user:ec2-user opensearch;', {
         cwd: '/home/ec2-user',
         ignoreErrors: false,
       }),
@@ -430,7 +432,6 @@
 
       fileContent['cluster.name'] = `${scope.stackName}-${scope.account}-${scope.region}`;
 
-      console.log(dump(fileContent).toString());
       opensearchConfig = dump(fileContent).toString();
       cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch; echo "${opensearchConfig}" > config/opensearch.yml`,
         {
@@ -464,7 +465,7 @@
           cwd: '/home/ec2-user',
           ignoreErrors: false,
         }));
         cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install repository-s3 --batch', {
           cwd: '/home/ec2-user',
           ignoreErrors: false,
         }));
@@ -476,13 +477,12 @@
           ignoreErrors: false,
         }));
         cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install '
-            + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}`
-            + `/tar/builds/opensearch/core-plugins/repository-s3-${props.opensearchVersion}.zip --batch`, {
+          + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}`
+          + `/tar/builds/opensearch/core-plugins/repository-s3-${props.opensearchVersion}.zip --batch`, {
           cwd: '/home/ec2-user',
           ignoreErrors: false,
         }));
       }
-
     }
 
     // add config to disable security if required
@@ -499,7 +499,7 @@
     if (props.jvmSysPropsString.toString() !== 'undefined') {
       // @ts-ignore
       cfnInitConfig.push(InitCommand.shellCommand(`set -ex; cd opensearch; jvmSysPropsList=$(echo "${props.jvmSysPropsString.toString()}" | tr ',' '\\n');`
-      + 'for sysProp in $jvmSysPropsList;do echo "-D$sysProp" >> config/jvm.options;done',
+        + 'for sysProp in $jvmSysPropsList;do echo "-D$sysProp" >> config/jvm.options;done',
       {
         cwd: '/home/ec2-user',
         ignoreErrors: false,
@@ -530,9 +530,10 @@
         }));
     }
 
-    // final run command based on whether the distribution type is min or bundle
+    // // Startinng OpenSearch based on whether the distribution type is min or bundle
     if (props.minDistribution) { // using (stackProps.minDistribution) condition is not working when false value is being sent
+      // eslint-disable-next-line max-len
       cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch; sudo -u ec2-user cp config/opensearch.yml config/elasticsearch.yml; sudo -u ec2-user nohup ./bin/opensearch >> install.log 2>&1 &',
         {
           cwd: '/home/ec2-user',
           ignoreErrors: false,
@@ -545,10 +546,10 @@
         }));
     }
 
-    // If OSD Url is present
+    // If OpenSearch-Dashboards URL is present
     if (props.dashboardsUrl !== 'undefined') {
       cfnInitConfig.push(InitCommand.shellCommand(`set -ex;mkdir opensearch-dashboards; curl -L ${props.dashboardsUrl} -o opensearch-dashboards.tar.gz;`
-          + 'tar zxf opensearch-dashboards.tar.gz -C opensearch-dashboards --strip-components=1; chown -R ec2-user:ec2-user opensearch-dashboards;', {
+        + 'tar zxf opensearch-dashboards.tar.gz -C opensearch-dashboards --strip-components=1; chown -R ec2-user:ec2-user opensearch-dashboards;', {
         cwd: '/home/ec2-user',
         ignoreErrors: false,
       }));
@@ -561,17 +562,28 @@
 
       if (props.securityDisabled && !props.minDistribution) {
         cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch-dashboards;'
-            + './bin/opensearch-dashboards-plugin remove securityDashboards --allow-root;'
-            + 'sed -i /^opensearch_security/d config/opensearch_dashboards.yml;'
-            + 'sed -i \'s/https/http/\' config/opensearch_dashboards.yml',
+          + './bin/opensearch-dashboards-plugin remove securityDashboards --allow-root;'
+          + 'sed -i /^opensearch_security/d config/opensearch_dashboards.yml;'
+          + 'sed -i \'s/https/http/\' config/opensearch_dashboards.yml',
         {
           cwd: '/home/ec2-user',
           ignoreErrors: false,
         }));
       }
 
+      // @ts-ignore
+      if (props.additionalOsdConfig.toString() !== 'undefined') {
+        // @ts-ignore
+        cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch-dashboards; echo "${props.additionalOsdConfig}">>config/opensearch_dashboards.yml`,
+          {
+            cwd: '/home/ec2-user',
+            ignoreErrors: false,
+          }));
+      }
+
+      // Startinng OpenSearch-Dashboards
       cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch-dashboards;'
-          + 'sudo -u ec2-user nohup ./bin/opensearch-dashboards > dashboard_install.log 2>&1 &', {
+        + 'sudo -u ec2-user nohup ./bin/opensearch-dashboards > dashboard_install.log 2>&1 &', {
         cwd: '/home/ec2-user',
         ignoreErrors: false,
       }));

lib/networking/vpc-stack.ts

@@ -5,14 +5,14 @@
 this file be licensed under the Apache-2.0 license or a
 compatible open source license. */
 
-import { Construct } from 'constructs';
+import { Stack, StackProps } from 'aws-cdk-lib';
 import {
   IPeer,
   ISecurityGroup,
   IVpc,
   Peer, Port, SecurityGroup, SubnetType, Vpc,
 } from 'aws-cdk-lib/aws-ec2';
-import { App, Stack, StackProps } from 'aws-cdk-lib';
+import { Construct } from 'constructs';
 
 export interface vpcProps extends StackProps{
     cidrBlock: string,
@@ -32,7 +32,7 @@
     let serverAccess: IPeer;
     super(scope, id, props);
     if (props.vpcId === undefined) {
-      console.log('No VPC Provided, creating new');
+      console.log('No VPC-Id Provided, a new VPC will be created');
       this.vpc = new Vpc(this, 'opensearchClusterVpc', {
         cidr: (props.cidrBlock !== undefined) ? props.cidrBlock : '10.0.0.0/16',
         maxAzs: props.maxAzs,
@@ -50,7 +50,7 @@
         ],
       });
     } else {
       console.log('VPC provided, using existing');
       this.vpc = Vpc.fromLookup(this, 'opensearchClusterVpc', {
         vpcId: props.vpcId,
       });

lib/os-cluster-entrypoint.ts

@@ -5,16 +5,22 @@
 this file be licensed under the Apache-2.0 license or a
 compatible open source license. */
 
-import { Construct } from 'constructs';
 import { Stack, StackProps } from 'aws-cdk-lib';
 import {
-  AmazonLinuxCpuType, InstanceType, IVpc, SecurityGroup, Vpc,
+  AmazonLinuxCpuType,
+  IVpc,
+  InstanceType,
+  SecurityGroup,
 } from 'aws-cdk-lib/aws-ec2';
+import { Construct } from 'constructs';
 import { dump } from 'js-yaml';
-import { NetworkStack } from './networking/vpc-stack';
 import { InfraStack } from './infra/infra-stack';
+import { NetworkStack } from './networking/vpc-stack';
 import {
-  x64Ec2InstanceType, arm64Ec2InstanceType, getX64InstanceTypes, getArm64InstanceTypes,
+  arm64Ec2InstanceType,
+  getArm64InstanceTypes,
+  getX64InstanceTypes,
+  x64Ec2InstanceType,
 } from './opensearch-config/node-config';
 
 enum cpuArchEnum{
@@ -53,6 +59,7 @@
       let dataNodeStorage: number;
       let mlNodeStorage: number;
       let ymlConfig: string = 'undefined';
+      let osdYmlConfig: string = 'undefined';
       let dataEc2InstanceType: InstanceType;
       let mlEc2InstanceType: InstanceType;
 
@@ -174,6 +181,16 @@
         }
       }
 
+      const osdConfig = `${scope.node.tryGetContext('additionalOsdConfig')}`;
+      if (osdConfig.toString() !== 'undefined') {
+        try {
+          const jsonObj = JSON.parse(osdConfig);
+          osdYmlConfig = dump(jsonObj);
+        } catch (e) {
+          throw new Error(`Encountered following error while parsing additionalOsdConfig json parameter: ${e}`);
+        }
+      }
+
       const suffix = `${scope.node.tryGetContext('suffix')}`;
 
       const use50heap = `${scope.node.tryGetContext('use50PercentHeap')}`;
@@ -230,6 +247,7 @@
         mlNodeStorage,
         jvmSysPropsString: jvmSysProps,
         additionalConfig: ymlConfig,
+        additionalOsdConfig: osdYmlConfig,
         use50PercentHeap,
         isInternal,
         customRoleArn,

test/os-cluster.test.ts

@@ -23,13 +23,16 @@ test('Test Resources with security disabled multi-node default instance types',
       serverAccessType: 'ipv4',
       restrictServerAccessTo: 'all',
       additionalConfig: '{ "name": "John Doe", "age": 30, "email": "[email protected]" }',
+      additionalOsdConfig: '{ "something.enabled": "true", "something_else.enabled": "false" }',
     },
   });
 
   // WHEN
   const securityDisabledStack = new OsClusterEntrypoint(app, {
     env: { account: 'test-account', region: 'us-east-1' },
   });
+
+  // THEN
   expect(securityDisabledStack.stacks).toHaveLength(2);
   const networkStack = securityDisabledStack.stacks.filter((s) => s.stackName === 'opensearch-network-stack')[0];
   const networkTemplate = Template.fromStack(networkStack);
@@ -285,6 +288,8 @@ test('Test multi-node cluster with only data-nodes', () => {
   const testStack = new OsClusterEntrypoint(app, {
     env: { account: 'test-account', region: 'us-east-1' },
   });
+
+  // THEN
   expect(testStack.stacks).toHaveLength(2);
 
   const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0];
@@ -329,6 +334,8 @@ test('Test multi-node cluster with custom IAM Role', () => {
   const testStack = new OsClusterEntrypoint(app, {
     env: { account: 'test-account', region: 'us-east-1' },
   });
+
+  // THEN
   expect(testStack.stacks).toHaveLength(2);
 
   const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0];