diff --git a/README.md b/README.md index f42f20f3bc7..b778b7b44a8 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,24 @@ # CDK for deploying single-node and multi-node OpenSearch cluster with OpenSearch Dashboards -- [Getting Started](#getting-started) -- [Deployment](#deployment) - - [Required context parameters](#required-context-parameters) - - [Interacting with OpenSearch cluster](#interacting-with-opensearch-cluster) - - [Restricting Server Access](#restricting-server-access) - - [Enable Remote Store Feature](#enable-remote-store-feature) -- [Check Logs](#check-logs) -- [Access EC2 Instances](#access-ec2-instances) -- [Port Mapping](#port-mapping) -- [Teardown](#teardown) -- [Contributing](#contributing) -- [Getting Help](#getting-help) -- [Code of Conduct](#code-of-conduct) -- [Security](#security) -- [License](#license) +- [CDK for deploying single-node and multi-node OpenSearch cluster with OpenSearch Dashboards](#cdk-for-deploying-single-node-and-multi-node-opensearch-cluster-with-opensearch-dashboards) + - [Getting Started](#getting-started) + - [Deployment](#deployment) + - [Required context parameters](#required-context-parameters) + - [Sample command to set up multi-node cluster with security enabled on x64 AL2 machine](#sample-command-to-set-up-multi-node-cluster-with-security-enabled-on-x64-al2-machine) + - [Interacting with OpenSearch cluster](#interacting-with-opensearch-cluster) + - [Sample commands](#sample-commands) + - [Restricting Server Access](#restricting-server-access) + - [Please note the load-balancer url is internet facing and can be accessed by anyone.](#please-note-the-load-balancer-url-is-internet-facing-and-can-be-accessed-by-anyone) + - [Enable Remote Store Feature](#enable-remote-store-feature) + - [Check logs](#check-logs) + - [Access EC2 Instances](#access-ec2-instances) + - [Port Mapping](#port-mapping) + - [Teardown](#teardown) + - [Contributing](#contributing) + - [Getting Help](#getting-help) + - [Code of Conduct](#code-of-conduct) + - [Security](#security) + - [License](#license) This project enables user to deploy either a single-node or a multi-node OpenSearch cluster. There are two stacks that get deployed: @@ -56,8 +60,8 @@ In order to deploy both the stacks the user needs to provide a set of required a | dataInstanceType | Optional | string | EC2 instance type for data node. Defaults to r5.xlarge. See options in `lib/opensearch-config/node-config.ts` for available options. E.g., `-c dataInstanceType=m5.xlarge` | | mlInstanceType | Optional | string | EC2 instance type for ml node. Defaults to r5.xlarge. See options in `lib/opensearch-config/node-config.ts` for available options. E.g., `-c mlInstanceType=m5.xlarge` | | jvmSysProps | Optional | string | A comma-separated list of key=value pairs that will be added to `jvm.options` as JVM system properties. | -| additionalConfig | Optional | string | Additional opensearch.yml config parameters passed as JSON. e.g., `--context additionalConfig='{"plugins.security.nodes_dn": ["CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE", "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"], "plugins.security.nodes_dn_dynamic_config_enabled": false}'` | -| additionalOsdConfig | Optional | string | Additional opensearch_dashboards.yml config parameters passed as JSON. e.g., `additionalOsdConfig='{"data.search.usageTelemetry.enabled": "true"}'` | +| additionalConfig | Optional | string | Additional opensearch.yml config parameters passed as JSON. Please be aware that this JSON merges with original opensearch.yml overwriting duplicate keys e.g., `--context additionalConfig='{"plugins.security.nodes_dn": ["CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE", "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"], "plugins.security.nodes_dn_dynamic_config_enabled": false}'` | +| additionalOsdConfig | Optional | string | Additional opensearch_dashboards.yml config parameters passed as JSON. Please be aware that this JSON merges with original opensearch-dashboards.yml overwriting duplicate keys. e.g., `additionalOsdConfig='{"data.search.usageTelemetry.enabled": "true"}'` | | suffix | Optional | string | An optional string identifier to be concatenated with infra stack name. | | networkStackSuffix (Optional) | string | An optional string identifier to be concatenated with network stack name. | | | region | Optional | string | User provided aws region | diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 364c3b34ad4..80fe0dc0560 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -657,6 +657,8 @@ export class InfraStack extends Stack { const cfnInitConfig: InitElement[] = [ InitPackage.yum('amazon-cloudwatch-agent'), + InitCommand.shellCommand('sudo wget -nv https://github.com/mikefarah/yq/releases/download/v4.40.5/yq_linux_amd64 ' + + '-O /usr/bin/yq && sudo chmod +x /usr/bin/yq'), CloudwatchAgent.asInitFile('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', { agent: { @@ -875,11 +877,12 @@ export class InfraStack extends Stack { } if (this.additionalConfig.toString() !== 'undefined') { - cfnInitConfig.push(InitCommand.shellCommand(`set -ex; cd opensearch; echo "${this.additionalConfig}">>config/opensearch.yml`, - { - cwd: '/home/ec2-user', - ignoreErrors: false, - })); + cfnInitConfig.push(InitCommand.shellCommand(`set -ex; cd opensearch/config; echo "${this.additionalConfig}">additionalConfig.yml; ` + + 'yq eval-all -i \'. as $item ireduce ({}; . * $item)\' opensearch.yml additionalConfig.yml -P', + { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); } if (this.customConfigFiles !== 'undefined') { @@ -907,12 +910,12 @@ export class InfraStack extends Stack { ignoreErrors: false, })); } else { - // eslint-disable-next-line max-len - cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch; sudo -u ec2-user nohup env OPENSEARCH_INITIAL_ADMIN_PASSWORD=${this.adminPassword} ./opensearch-tar-install.sh >> install.log 2>&1 &`, - { - cwd: '/home/ec2-user', - ignoreErrors: false, - })); + cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch; ' + + `sudo -u ec2-user nohup env OPENSEARCH_INITIAL_ADMIN_PASSWORD=${this.adminPassword} ./opensearch-tar-install.sh >> install.log 2>&1 &`, + { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); } // If OpenSearch-Dashboards URL is present @@ -941,11 +944,12 @@ export class InfraStack extends Stack { } if (this.additionalOsdConfig.toString() !== 'undefined') { - cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch-dashboards; echo "${this.additionalOsdConfig}">>config/opensearch_dashboards.yml`, - { - cwd: '/home/ec2-user', - ignoreErrors: false, - })); + cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch-dashboards/config; echo "${this.additionalOsdConfig}">additionalOsdConfig.yml; ` + + 'yq eval-all -i \'. as $item ireduce ({}; . * $item)\' opensearch_dashboards.yml additionalOsdConfig.yml -P', + { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); } // Starting OpenSearch-Dashboards diff --git a/package-lock.json b/package-lock.json index dd0faab2b3b..8795e37a8c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@opensearch-project/opensearch-cluster-cdk", - "version": "1.0.1", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@opensearch-project/opensearch-cluster-cdk", - "version": "1.0.1", + "version": "1.1.0", "dependencies": { "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", diff --git a/package.json b/package.json index a12d695a170..91d87757c8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@opensearch-project/opensearch-cluster-cdk", - "version": "1.0.1", + "version": "1.1.0", "bin": { "cdk_v2": "bin/app.js" }, diff --git a/test/opensearch-cluster-cdk.test.ts b/test/opensearch-cluster-cdk.test.ts index 290e99922d8..62ae2046efb 100644 --- a/test/opensearch-cluster-cdk.test.ts +++ b/test/opensearch-cluster-cdk.test.ts @@ -768,3 +768,62 @@ test('Should not throw error when security is enabled and version is less than 2 env: { account: 'test-account', region: 'us-east-1' }, }); }); + +test('Test additionalConfig overriding values', () => { + const app = new App({ + context: { + securityDisabled: true, + minDistribution: false, + distributionUrl: 'www.example.com', + cpuArch: 'x64', + singleNodeCluster: false, + dashboardsUrl: 'www.example.com', + distVersion: '1.0.0', + serverAccessType: 'ipv4', + restrictServerAccessTo: 'all', + managerNodeCount: 0, + dataNodeCount: 3, + dataNodeStorage: 200, + customRoleArn: 'arn:aws:iam::12345678:role/customRoleName', + additionalConfig: '{ "cluster.name": "custom-cdk", "network.port": "8041" }', + additionalOsdConfig: '{ "something.enabled": "true", "something_else.enabled": "false" }', + }, + }); + + // WHEN + const networkStack = new NetworkStack(app, 'opensearch-network-stack', { + env: { account: 'test-account', region: 'us-east-1' }, + }); + + // @ts-ignore + const infraStack = new InfraStack(app, 'opensearch-infra-stack', { + vpc: networkStack.vpc, + securityGroup: networkStack.osSecurityGroup, + env: { account: 'test-account', region: 'us-east-1' }, + }); + + // THEN + const infraTemplate = Template.fromStack(infraStack); + infraTemplate.resourceCountIs('AWS::IAM::Role', 0); + infraTemplate.hasResource('AWS::AutoScaling::AutoScalingGroup', { + /* eslint-disable max-len */ + Metadata: { + 'AWS::CloudFormation::Init': { + config: { + commands: { + '011': { + command: "set -ex; cd opensearch/config; echo \"cluster.name: custom-cdk\nnetwork.port: '8041'\n\">additionalConfig.yml; yq eval-all -i '. as $item ireduce ({}; . * $item)' opensearch.yml additionalConfig.yml -P", + cwd: '/home/ec2-user', + ignoreErrors: false, + }, + '016': { + command: "set -ex;cd opensearch-dashboards/config; echo \"something.enabled: 'true'\nsomething_else.enabled: 'false'\n\">additionalOsdConfig.yml; yq eval-all -i '. as $item ireduce ({}; . * $item)' opensearch_dashboards.yml additionalOsdConfig.yml -P", + cwd: '/home/ec2-user', + ignoreErrors: false, + }, + }, + }, + }, + }, + }); +});