From bec6d291bb734d11625eac7b47b5d8bd99ec6a70 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:14:28 -0700 Subject: [PATCH] Add support to add additional configuration to opensearch-dashboards.yml (#77) Signed-off-by: Sayali Gaikawad --- README.md | 1 + lib/infra/infra-stack.ts | 96 ++++++++++++++++++++---------------- lib/networking/vpc-stack.ts | 6 +-- lib/os-cluster-entrypoint.ts | 23 +++++++-- test/os-cluster.test.ts | 9 ++++ 5 files changed, 86 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 5f3a860c145..e305986fb9a 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ In order to deploy both the stacks the user needs to provide a set of required a | mlInstanceType | Optional | string | EC2 instance type for ml node. Defaults to r5.xlarge. See options in `lib/opensearch-config/node-config.ts` for available options. E.g., `-c mlInstanceType=m5.xlarge` | | jvmSysProps | Optional | string | A comma-separated list of key=value pairs that will be added to `jvm.options` as JVM system properties. | | additionalConfig | Optional | string | Additional opensearch.yml config parameters passed as JSON. e.g., `--context additionalConfig='{"plugins.security.nodes_dn": ["CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE", "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"], "plugins.security.nodes_dn_dynamic_config_enabled": false}'` | +| additionalOsdConfig | Optional | string | Additional opensearch_dashboards.yml config parameters passed as JSON. e.g., `additionalOsdConfig='{"data.search.usageTelemetry.enabled": "true"}'` | | suffix | Optional | string | An optional string identifier to be concatenated with infra stack name. | | region | Optional | string | User provided aws region | | account | Optional | string | User provided aws account | diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 0976e2f56b1..c9d260dc2d1 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -30,7 +30,8 @@ import { import { NetworkListener, NetworkLoadBalancer, Protocol } from 'aws-cdk-lib/aws-elasticloadbalancingv2'; import { InstanceTarget } from 'aws-cdk-lib/aws-elasticloadbalancingv2-targets'; import { - ManagedPolicy, Role, IRole, ServicePrincipal, + ManagedPolicy, Role, + ServicePrincipal, } from 'aws-cdk-lib/aws-iam'; import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { readFileSync } from 'fs'; @@ -41,32 +42,33 @@ import { nodeConfig } from '../opensearch-config/node-config'; import { RemoteStoreResources } from './remote-store-resources'; export interface infraProps extends StackProps { - readonly vpc: IVpc, - readonly securityGroup: ISecurityGroup, - readonly opensearchVersion: string, - readonly cpuArch: string, - readonly cpuType: AmazonLinuxCpuType, - readonly securityDisabled: boolean, - readonly minDistribution: boolean, - readonly distributionUrl: string, - readonly dashboardsUrl: string, - readonly singleNodeCluster: boolean, - readonly managerNodeCount: number, - readonly dataNodeCount: number, - readonly ingestNodeCount: number, - readonly clientNodeCount: number, - readonly mlNodeCount: number, - readonly dataNodeStorage: number, - readonly mlNodeStorage: number, - readonly jvmSysPropsString?: string, - readonly additionalConfig?: string, - readonly dataEc2InstanceType: InstanceType, - readonly mlEc2InstanceType: InstanceType, - readonly use50PercentHeap: boolean, - readonly isInternal: boolean, - readonly enableRemoteStore: boolean, - readonly storageVolumeType: EbsDeviceVolumeType, - readonly customRoleArn: string + readonly vpc: IVpc, + readonly securityGroup: ISecurityGroup, + readonly opensearchVersion: string, + readonly cpuArch: string, + readonly cpuType: AmazonLinuxCpuType, + readonly securityDisabled: boolean, + readonly minDistribution: boolean, + readonly distributionUrl: string, + readonly dashboardsUrl: string, + readonly singleNodeCluster: boolean, + readonly managerNodeCount: number, + readonly dataNodeCount: number, + readonly ingestNodeCount: number, + readonly clientNodeCount: number, + readonly mlNodeCount: number, + readonly dataNodeStorage: number, + readonly mlNodeStorage: number, + readonly jvmSysPropsString?: string, + readonly additionalConfig?: string, + readonly additionalOsdConfig?: string, + readonly dataEc2InstanceType: InstanceType, + readonly mlEc2InstanceType: InstanceType, + readonly use50PercentHeap: boolean, + readonly isInternal: boolean, + readonly enableRemoteStore: boolean, + readonly storageVolumeType: EbsDeviceVolumeType, + readonly customRoleArn: string } export class InfraStack extends Stack { @@ -373,7 +375,7 @@ export class InfraStack extends Stack { const configFileDir = join(__dirname, '../opensearch-config'); let opensearchConfig: string; - const cfnInitConfig : InitElement[] = [ + const cfnInitConfig: InitElement[] = [ InitPackage.yum('amazon-cloudwatch-agent'), CloudwatchAgent.asInitFile('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', { @@ -435,7 +437,7 @@ export class InfraStack extends Stack { InitCommand.shellCommand('set -ex;/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s'), InitCommand.shellCommand('set -ex; sudo echo "vm.max_map_count=262144" >> /etc/sysctl.conf;sudo sysctl -p'), InitCommand.shellCommand(`set -ex;mkdir opensearch; curl -L ${props.distributionUrl} -o opensearch.tar.gz;` - + 'tar zxf opensearch.tar.gz -C opensearch --strip-components=1; chown -R ec2-user:ec2-user opensearch;', { + + 'tar zxf opensearch.tar.gz -C opensearch --strip-components=1; chown -R ec2-user:ec2-user opensearch;', { cwd: '/home/ec2-user', ignoreErrors: false, }), @@ -448,7 +450,6 @@ export class InfraStack extends Stack { fileContent['cluster.name'] = `${scope.stackName}-${scope.account}-${scope.region}`; - console.log(dump(fileContent).toString()); opensearchConfig = dump(fileContent).toString(); cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch; echo "${opensearchConfig}" > config/opensearch.yml`, { @@ -488,14 +489,14 @@ export class InfraStack extends Stack { })); } else { cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install ' - + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` - + `/tar/builds/opensearch/core-plugins/discovery-ec2-${props.opensearchVersion}.zip --batch`, { + + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` + + `/tar/builds/opensearch/core-plugins/discovery-ec2-${props.opensearchVersion}.zip --batch`, { cwd: '/home/ec2-user', ignoreErrors: false, })); cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install ' - + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` - + `/tar/builds/opensearch/core-plugins/repository-s3-${props.opensearchVersion}.zip --batch`, { + + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` + + `/tar/builds/opensearch/core-plugins/repository-s3-${props.opensearchVersion}.zip --batch`, { cwd: '/home/ec2-user', ignoreErrors: false, })); @@ -549,7 +550,7 @@ export class InfraStack extends Stack { if (props.jvmSysPropsString.toString() !== 'undefined') { // @ts-ignore cfnInitConfig.push(InitCommand.shellCommand(`set -ex; cd opensearch; jvmSysPropsList=$(echo "${props.jvmSysPropsString.toString()}" | tr ',' '\\n');` - + 'for sysProp in $jvmSysPropsList;do echo "-D$sysProp" >> config/jvm.options;done', + + 'for sysProp in $jvmSysPropsList;do echo "-D$sysProp" >> config/jvm.options;done', { cwd: '/home/ec2-user', ignoreErrors: false, @@ -580,7 +581,7 @@ export class InfraStack extends Stack { })); } - // final run command based on whether the distribution type is min or bundle + // // Startinng OpenSearch based on whether the distribution type is min or bundle if (props.minDistribution) { // using (stackProps.minDistribution) condition is not working when false value is being sent cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch; sudo -u ec2-user nohup ./bin/opensearch >> install.log 2>&1 &', { @@ -595,10 +596,10 @@ export class InfraStack extends Stack { })); } - // If OSD Url is present + // If OpenSearch-Dashboards URL is present if (props.dashboardsUrl !== 'undefined') { cfnInitConfig.push(InitCommand.shellCommand(`set -ex;mkdir opensearch-dashboards; curl -L ${props.dashboardsUrl} -o opensearch-dashboards.tar.gz;` - + 'tar zxf opensearch-dashboards.tar.gz -C opensearch-dashboards --strip-components=1; chown -R ec2-user:ec2-user opensearch-dashboards;', { + + 'tar zxf opensearch-dashboards.tar.gz -C opensearch-dashboards --strip-components=1; chown -R ec2-user:ec2-user opensearch-dashboards;', { cwd: '/home/ec2-user', ignoreErrors: false, })); @@ -611,17 +612,28 @@ export class InfraStack extends Stack { if (props.securityDisabled && !props.minDistribution) { cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch-dashboards;' - + './bin/opensearch-dashboards-plugin remove securityDashboards --allow-root;' - + 'sed -i /^opensearch_security/d config/opensearch_dashboards.yml;' - + 'sed -i \'s/https/http/\' config/opensearch_dashboards.yml', + + './bin/opensearch-dashboards-plugin remove securityDashboards --allow-root;' + + 'sed -i /^opensearch_security/d config/opensearch_dashboards.yml;' + + 'sed -i \'s/https/http/\' config/opensearch_dashboards.yml', { cwd: '/home/ec2-user', ignoreErrors: false, })); } + // @ts-ignore + if (props.additionalOsdConfig.toString() !== 'undefined') { + // @ts-ignore + cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch-dashboards; echo "${props.additionalOsdConfig}">>config/opensearch_dashboards.yml`, + { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); + } + + // Startinng OpenSearch-Dashboards cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch-dashboards;' - + 'sudo -u ec2-user nohup ./bin/opensearch-dashboards > dashboard_install.log 2>&1 &', { + + 'sudo -u ec2-user nohup ./bin/opensearch-dashboards > dashboard_install.log 2>&1 &', { cwd: '/home/ec2-user', ignoreErrors: false, })); diff --git a/lib/networking/vpc-stack.ts b/lib/networking/vpc-stack.ts index 361cfa33810..e6af873b82e 100644 --- a/lib/networking/vpc-stack.ts +++ b/lib/networking/vpc-stack.ts @@ -5,14 +5,14 @@ The OpenSearch Contributors require contributions made to this file be licensed under the Apache-2.0 license or a compatible open source license. */ -import { Construct } from 'constructs'; +import { Stack, StackProps } from 'aws-cdk-lib'; import { IPeer, ISecurityGroup, IVpc, Peer, Port, SecurityGroup, SubnetType, Vpc, } from 'aws-cdk-lib/aws-ec2'; -import { App, Stack, StackProps } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; export interface vpcProps extends StackProps{ cidrBlock: string, @@ -32,7 +32,7 @@ export class NetworkStack extends Stack { let serverAccess: IPeer; super(scope, id, props); if (props.vpcId === undefined) { - console.log('No VPC Provided, creating new'); + console.log('No VPC-Id Provided, a new VPC will be created'); this.vpc = new Vpc(this, 'opensearchClusterVpc', { cidr: (props.cidrBlock !== undefined) ? props.cidrBlock : '10.0.0.0/16', maxAzs: props.maxAzs, diff --git a/lib/os-cluster-entrypoint.ts b/lib/os-cluster-entrypoint.ts index 752696b9bfa..071422f3305 100644 --- a/lib/os-cluster-entrypoint.ts +++ b/lib/os-cluster-entrypoint.ts @@ -5,15 +5,18 @@ The OpenSearch Contributors require contributions made to this file be licensed under the Apache-2.0 license or a compatible open source license. */ -import { Construct } from 'constructs'; import { Stack, StackProps } from 'aws-cdk-lib'; +import { EbsDeviceVolumeType } from 'aws-cdk-lib/aws-autoscaling'; import { - AmazonLinuxCpuType, InstanceType, IVpc, SecurityGroup, Vpc, + AmazonLinuxCpuType, + IVpc, + InstanceType, + SecurityGroup, } from 'aws-cdk-lib/aws-ec2'; +import { Construct } from 'constructs'; import { dump } from 'js-yaml'; -import { EbsDeviceVolumeType } from 'aws-cdk-lib/aws-autoscaling'; -import { NetworkStack } from './networking/vpc-stack'; import { InfraStack } from './infra/infra-stack'; +import { NetworkStack } from './networking/vpc-stack'; import { arm64Ec2InstanceType, getArm64InstanceTypes, @@ -58,6 +61,7 @@ export class OsClusterEntrypoint { let dataNodeStorage: number; let mlNodeStorage: number; let ymlConfig: string = 'undefined'; + let osdYmlConfig: string = 'undefined'; let dataEc2InstanceType: InstanceType; let mlEc2InstanceType: InstanceType; let volumeType: EbsDeviceVolumeType; @@ -188,6 +192,16 @@ export class OsClusterEntrypoint { } } + const osdConfig = `${scope.node.tryGetContext('additionalOsdConfig')}`; + if (osdConfig.toString() !== 'undefined') { + try { + const jsonObj = JSON.parse(osdConfig); + osdYmlConfig = dump(jsonObj); + } catch (e) { + throw new Error(`Encountered following error while parsing additionalOsdConfig json parameter: ${e}`); + } + } + const suffix = `${scope.node.tryGetContext('suffix')}`; const use50heap = `${scope.node.tryGetContext('use50PercentHeap')}`; @@ -247,6 +261,7 @@ export class OsClusterEntrypoint { mlNodeStorage, jvmSysPropsString: jvmSysProps, additionalConfig: ymlConfig, + additionalOsdConfig: osdYmlConfig, use50PercentHeap, isInternal, enableRemoteStore, diff --git a/test/os-cluster.test.ts b/test/os-cluster.test.ts index bf07428a824..b3b1eceb2bd 100644 --- a/test/os-cluster.test.ts +++ b/test/os-cluster.test.ts @@ -23,6 +23,7 @@ test('Test Resources with security disabled multi-node default instance types', serverAccessType: 'ipv4', restrictServerAccessTo: 'all', additionalConfig: '{ "name": "John Doe", "age": 30, "email": "johndoe@example.com" }', + additionalOsdConfig: '{ "something.enabled": "true", "something_else.enabled": "false" }', }, }); @@ -30,6 +31,8 @@ test('Test Resources with security disabled multi-node default instance types', const securityDisabledStack = new OsClusterEntrypoint(app, { env: { account: 'test-account', region: 'us-east-1' }, }); + + // THEN expect(securityDisabledStack.stacks).toHaveLength(2); const networkStack = securityDisabledStack.stacks.filter((s) => s.stackName === 'opensearch-network-stack')[0]; const networkTemplate = Template.fromStack(networkStack); @@ -299,6 +302,8 @@ test('Test multi-node cluster with only data-nodes', () => { const testStack = new OsClusterEntrypoint(app, { env: { account: 'test-account', region: 'us-east-1' }, }); + + // THEN expect(testStack.stacks).toHaveLength(2); const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0]; @@ -349,6 +354,8 @@ test('Test multi-node cluster with remote-store enabled', () => { const testStack = new OsClusterEntrypoint(app, { env: { account: 'test-account', region: 'us-east-1' }, }); + + // THEN expect(testStack.stacks).toHaveLength(2); const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0]; @@ -471,6 +478,8 @@ test('Test multi-node cluster with custom IAM Role', () => { const testStack = new OsClusterEntrypoint(app, { env: { account: 'test-account', region: 'us-east-1' }, }); + + // THEN expect(testStack.stacks).toHaveLength(2); const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0];