From 105440561d4454ee505dd598483775258b87aa0a Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 2 Mar 2023 17:17:08 -0600 Subject: [PATCH 01/33] Adding KMS keys to EFS CSI driver and updating IAM policy to allow encryption/decryption --- lib/addons/efs-csi-driver/iam-policy.ts | 104 ++++++++++++++++-------- lib/addons/efs-csi-driver/index.ts | 17 ++-- 2 files changed, 83 insertions(+), 38 deletions(-) diff --git a/lib/addons/efs-csi-driver/iam-policy.ts b/lib/addons/efs-csi-driver/iam-policy.ts index ffbddd123..c354ea0e9 100644 --- a/lib/addons/efs-csi-driver/iam-policy.ts +++ b/lib/addons/efs-csi-driver/iam-policy.ts @@ -1,37 +1,77 @@ -export const EfsDriverPolicyDocument = () => { - return { - "Version": "2012-10-17", - "Statement": [ +import * as kms from "aws-cdk-lib/aws-kms"; + +interface Statement { + Effect: string; + Action: string | string[]; + Resource: string | string[]; + Condition?: { + StringEquals?: { [key: string]: string[] | string }; + StringLike?: { [key: string]: string }; + Bool?: { [key: string]: string }; + }; +} + +export function getEfsDriverPolicyStatements( + kmsKeys?: kms.Key[] +): Statement[] { + const result: Statement[] = [ + { + "Effect": "Allow", + "Action": [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticfilesystem:CreateAccessPoint" + ], + "Resource": "*", + "Condition": { + "StringLike": { + "aws:RequestTag/efs.csi.aws.com/cluster": "true" + } + } + }, + { + "Effect": "Allow", + "Action": "elasticfilesystem:DeleteAccessPoint", + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/efs.csi.aws.com/cluster": "true" + } + } + } + ]; + if (kmsKeys) { + const kmsKeysArns = kmsKeys.map((k) => k.keyArn); + const kmsPolicy: Statement[] = [ { - "Effect": "Allow", - "Action": [ - "elasticfilesystem:DescribeAccessPoints", - "elasticfilesystem:DescribeFileSystems" - ], - "Resource": "*" + Effect: "Allow", + Action: ["kms:CreateGrant", "kms:ListGrants", "kms:RevokeGrant"], + Resource: kmsKeysArns, + Condition: { + Bool: { + "kms:GrantIsForAWSResource": "true", + }, + }, }, { - "Effect": "Allow", - "Action": [ - "elasticfilesystem:CreateAccessPoint" + Effect: "Allow", + Action: [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", ], - "Resource": "*", - "Condition": { - "StringLike": { - "aws:RequestTag/efs.csi.aws.com/cluster": "true" - } - } + Resource: kmsKeysArns, }, - { - "Effect": "Allow", - "Action": "elasticfilesystem:DeleteAccessPoint", - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:ResourceTag/efs.csi.aws.com/cluster": "true" - } - } - } - ] - }; -}; \ No newline at end of file + ]; + result.push(...kmsPolicy); + } + return result; +} diff --git a/lib/addons/efs-csi-driver/index.ts b/lib/addons/efs-csi-driver/index.ts index 4ae5e836c..1dba60af5 100644 --- a/lib/addons/efs-csi-driver/index.ts +++ b/lib/addons/efs-csi-driver/index.ts @@ -1,10 +1,11 @@ +import * as iam from "aws-cdk-lib/aws-iam"; +import * as kms from "aws-cdk-lib/aws-kms"; import { Construct } from "constructs"; -import {ClusterInfo, Values} from "../../spi"; +import { ClusterInfo, Values } from "../../spi"; +import { setPath } from "../../utils"; +import { registries } from "../../utils/registry-utils"; import { HelmAddOn, HelmAddOnUserProps } from "../helm-addon"; -import { EfsDriverPolicyDocument } from "./iam-policy"; -import { registries } from "../../utils/registry-utils"; -import * as iam from "aws-cdk-lib/aws-iam"; -import {setPath} from "../../utils"; +import { getEfsDriverPolicyStatements } from "./iam-policy"; const EFS_CSI_DRIVER = "aws-efs-csi-driver"; @@ -25,6 +26,10 @@ export interface EfsCsiDriverProps extends HelmAddOnUserProps { * pods will be left of pending state */ replicaCount?: number + /** + * List of KMS keys to be used for encryption + */ + kmsKeys?: kms.Key[]; } @@ -56,7 +61,7 @@ export class EfsCsiDriverAddOn extends HelmAddOn { name: EFS_CSI_CONTROLLER_SA, namespace: this.options.namespace, }); - EfsDriverPolicyDocument().Statement.forEach((statement) => { + getEfsDriverPolicyStatements().forEach((statement) => { serviceAccount.addToPrincipalPolicy(iam.PolicyStatement.fromJson(statement)); }); From 3af80ed385eedffa6f388074e985f676aa6cabf1 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 2 Mar 2023 18:50:24 -0600 Subject: [PATCH 02/33] Passing kmsKeys to the policy generator --- lib/addons/efs-csi-driver/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/addons/efs-csi-driver/index.ts b/lib/addons/efs-csi-driver/index.ts index 1dba60af5..1b05b51a3 100644 --- a/lib/addons/efs-csi-driver/index.ts +++ b/lib/addons/efs-csi-driver/index.ts @@ -61,7 +61,7 @@ export class EfsCsiDriverAddOn extends HelmAddOn { name: EFS_CSI_CONTROLLER_SA, namespace: this.options.namespace, }); - getEfsDriverPolicyStatements().forEach((statement) => { + getEfsDriverPolicyStatements(this.options?.kmsKeys).forEach((statement) => { serviceAccount.addToPrincipalPolicy(iam.PolicyStatement.fromJson(statement)); }); From 57b0ab18efea23c7de7b8ae72443c68354e4cbeb Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Tue, 7 Mar 2023 15:48:29 -0600 Subject: [PATCH 03/33] Adding EFS resource provider --- lib/resource-providers/efs.ts | 81 +++++++++++++++++++++++++++++++++ lib/resource-providers/index.ts | 5 +- lib/spi/types.ts | 1 + 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 lib/resource-providers/efs.ts diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts new file mode 100644 index 000000000..591727191 --- /dev/null +++ b/lib/resource-providers/efs.ts @@ -0,0 +1,81 @@ +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as efs from 'aws-cdk-lib/aws-efs'; +import * as cdk from 'aws-cdk-lib/core'; +import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; + +/** + * EFS resource provider + * Pass either an EFS file system name to lookup an existing or create a new one. + * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target, if omitted a file system will be created. + * @param removalPolicy The removal policy to use for the EFS file system + * @param efsProps The key props used + */ +export class EfsFileSystemProvider implements ResourceProvider { + readonly name?: string; + readonly efsProps?: efs.FileSystemProps; + readonly removalPolicy?: cdk.RemovalPolicy; + + constructor(name?: string, efsProps?: efs.FileSystemProps, removalPolicy?: cdk.RemovalPolicy) { + this.name = name; + this.efsProps = efsProps; + this.removalPolicy = removalPolicy; + } + + provide(context: ResourceContext): efs.IFileSystem { + const id = context.scope.node.id; + const securityGroupId = `${id}-${this.name}-EfsSecurityGroup` || `${id}-EfsSecurityGroup`; + let efsFileSystem = undefined; + + if (this.name) { + const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( + context.scope, + securityGroupId, + securityGroupId, + ); + efsFileSystem = efs.FileSystem.fromFileSystemAttributes( + context.scope, this.name, + { + securityGroup: securityGroup, + } + ); + } + + if (!efsFileSystem) { + const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; + const removalPolicy = this.removalPolicy || context.removalPolicy; + const clusterVpcCidr = vpc.vpcCidrBlock; + if (!vpc) { + throw new Error('VPC not found in context'); + } + + const efsSG = new ec2.SecurityGroup( + context.scope, securityGroupId, + { + vpc: vpc, + securityGroupName: securityGroupId, + } + ); + efsSG.addIngressRule( + ec2.Peer.ipv4(clusterVpcCidr), + new ec2.Port({ + protocol: ec2.Protocol.TCP, + stringRepresentation: "EFSconnection", + toPort: 2049, + fromPort: 2049, + }), + ); + + efsFileSystem = new efs.FileSystem( + context.scope, this.name || `${id}-EfsFileSystem`, + { + vpc: vpc, + securityGroup: efsSG, + removalPolicy: removalPolicy, + encrypted: true, + ...this.efsProps, + } + ); + } + return efsFileSystem; + } +} diff --git a/lib/resource-providers/index.ts b/lib/resource-providers/index.ts index c64f75d03..2f9dc3485 100644 --- a/lib/resource-providers/index.ts +++ b/lib/resource-providers/index.ts @@ -1,6 +1,7 @@ export * from './certificate'; +export * from './efs'; export * from './hosted-zone'; -export * from './kms-key'; export * from './iam'; +export * from './kms-key'; export * from './utils'; -export * from './vpc'; \ No newline at end of file +export * from './vpc'; diff --git a/lib/spi/types.ts b/lib/spi/types.ts index 7515e41fd..5a2bd125d 100644 --- a/lib/spi/types.ts +++ b/lib/spi/types.ts @@ -80,6 +80,7 @@ export interface ApplicationRepository extends GitRepositoryReference { export class ResourceContext { private readonly resources: Map = new Map(); + removalPolicy: cdk.RemovalPolicy | undefined; constructor(public readonly scope: cdk.Stack, public readonly blueprintProps: EksBlueprintProps) { } From 1496c622149ffff4a0381720876358fe0a8295de Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Tue, 7 Mar 2023 17:26:21 -0600 Subject: [PATCH 04/33] Adding interface for EFS resource provider --- lib/resource-providers/efs.ts | 62 +++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 591727191..a29d9501f 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -4,57 +4,61 @@ import * as cdk from 'aws-cdk-lib/core'; import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; /** - * EFS resource provider - * Pass either an EFS file system name to lookup an existing or create a new one. - * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target, if omitted a file system will be created. - * @param removalPolicy The removal policy to use for the EFS file system - * @param efsProps The key props used + * Configuration options for the EFS file system. */ -export class EfsFileSystemProvider implements ResourceProvider { +export interface EfsFileSystemProps { readonly name?: string; - readonly efsProps?: efs.FileSystemProps; + readonly fileSystemId?: string; + readonly efsProps?: Omit; readonly removalPolicy?: cdk.RemovalPolicy; +} + +/** + * EFS resource provider. + * + * Pass either an EFS file system name and id to lookup an existing one, or create a new one. + * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + * @param fileSystemId The id of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + * @param efsProps The props used for the file system. + * @param removalPolicy The removal policy to use for the EFS file system. + */ +export class EfsFileSystemProvider implements ResourceProvider { + readonly options: EfsFileSystemProps; - constructor(name?: string, efsProps?: efs.FileSystemProps, removalPolicy?: cdk.RemovalPolicy) { - this.name = name; - this.efsProps = efsProps; - this.removalPolicy = removalPolicy; + constructor(options: EfsFileSystemProps) { + this.options = options; } provide(context: ResourceContext): efs.IFileSystem { const id = context.scope.node.id; - const securityGroupId = `${id}-${this.name}-EfsSecurityGroup` || `${id}-EfsSecurityGroup`; - let efsFileSystem = undefined; + const securityGroupId = `${id}-${this.options.name ?? 'default'}-EfsSecurityGroup`; + let efsFileSystem: efs.IFileSystem | undefined; - if (this.name) { + if (this.options.fileSystemId && this.options.name) { const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( context.scope, securityGroupId, securityGroupId, ); efsFileSystem = efs.FileSystem.fromFileSystemAttributes( - context.scope, this.name, + context.scope, this.options.name, { securityGroup: securityGroup, + fileSystemId: this.options.fileSystemId, } ); } if (!efsFileSystem) { const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; - const removalPolicy = this.removalPolicy || context.removalPolicy; - const clusterVpcCidr = vpc.vpcCidrBlock; - if (!vpc) { + + if (vpc === undefined) { throw new Error('VPC not found in context'); } + const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; + const clusterVpcCidr = vpc.vpcCidrBlock; - const efsSG = new ec2.SecurityGroup( - context.scope, securityGroupId, - { - vpc: vpc, - securityGroupName: securityGroupId, - } - ); + const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { vpc, securityGroupName: securityGroupId, }); efsSG.addIngressRule( ec2.Peer.ipv4(clusterVpcCidr), new ec2.Port({ @@ -66,13 +70,13 @@ export class EfsFileSystemProvider implements ResourceProvider ); efsFileSystem = new efs.FileSystem( - context.scope, this.name || `${id}-EfsFileSystem`, + context.scope, this.options.name || `${id}-EfsFileSystem`, { - vpc: vpc, + vpc, securityGroup: efsSG, - removalPolicy: removalPolicy, + removalPolicy, encrypted: true, - ...this.efsProps, + ...this.options.efsProps, } ); } From 25eb3de15afc1e49a4eb7f0f88a09ba37399b6ff Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Tue, 7 Mar 2023 19:28:40 -0600 Subject: [PATCH 05/33] Delegating encryption config to the pattern --- lib/resource-providers/efs.ts | 123 ++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 58 deletions(-) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index a29d9501f..785717085 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -1,16 +1,16 @@ -import * as ec2 from 'aws-cdk-lib/aws-ec2'; -import * as efs from 'aws-cdk-lib/aws-efs'; -import * as cdk from 'aws-cdk-lib/core'; +import * as ec2 from "aws-cdk-lib/aws-ec2"; +import * as efs from "aws-cdk-lib/aws-efs"; +import * as cdk from "aws-cdk-lib/core"; import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; /** * Configuration options for the EFS file system. */ export interface EfsFileSystemProps { - readonly name?: string; - readonly fileSystemId?: string; - readonly efsProps?: Omit; - readonly removalPolicy?: cdk.RemovalPolicy; + readonly name?: string; + readonly fileSystemId?: string; + readonly efsProps?: Omit; + readonly removalPolicy?: cdk.RemovalPolicy; } /** @@ -22,64 +22,71 @@ export interface EfsFileSystemProps { * @param efsProps The props used for the file system. * @param removalPolicy The removal policy to use for the EFS file system. */ -export class EfsFileSystemProvider implements ResourceProvider { - readonly options: EfsFileSystemProps; +export class EfsFileSystemProvider + implements ResourceProvider +{ + readonly options: EfsFileSystemProps; - constructor(options: EfsFileSystemProps) { - this.options = options; - } + constructor(options: EfsFileSystemProps) { + this.options = options; + } - provide(context: ResourceContext): efs.IFileSystem { - const id = context.scope.node.id; - const securityGroupId = `${id}-${this.options.name ?? 'default'}-EfsSecurityGroup`; - let efsFileSystem: efs.IFileSystem | undefined; + provide(context: ResourceContext): efs.IFileSystem { + const id = context.scope.node.id; + const securityGroupId = `${id}-${ + this.options.name ?? "default" + }-EfsSecurityGroup`; + let efsFileSystem: efs.IFileSystem | undefined; - if (this.options.fileSystemId && this.options.name) { - const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( - context.scope, - securityGroupId, - securityGroupId, - ); - efsFileSystem = efs.FileSystem.fromFileSystemAttributes( - context.scope, this.options.name, - { - securityGroup: securityGroup, - fileSystemId: this.options.fileSystemId, - } - ); + if (this.options.fileSystemId && this.options.name) { + const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( + context.scope, + securityGroupId, + securityGroupId + ); + efsFileSystem = efs.FileSystem.fromFileSystemAttributes( + context.scope, + this.options.name, + { + securityGroup: securityGroup, + fileSystemId: this.options.fileSystemId, } + ); + } - if (!efsFileSystem) { - const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; - - if (vpc === undefined) { - throw new Error('VPC not found in context'); - } - const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; - const clusterVpcCidr = vpc.vpcCidrBlock; + if (!efsFileSystem) { + const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; + if (vpc === undefined) { + throw new Error("VPC not found in context"); + } + const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; + const clusterVpcCidr = vpc.vpcCidrBlock; - const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { vpc, securityGroupName: securityGroupId, }); - efsSG.addIngressRule( - ec2.Peer.ipv4(clusterVpcCidr), - new ec2.Port({ - protocol: ec2.Protocol.TCP, - stringRepresentation: "EFSconnection", - toPort: 2049, - fromPort: 2049, - }), - ); + const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { + vpc, + securityGroupName: securityGroupId, + }); + efsSG.addIngressRule( + ec2.Peer.ipv4(clusterVpcCidr), + new ec2.Port({ + protocol: ec2.Protocol.TCP, + stringRepresentation: "EFSconnection", + toPort: 2049, + fromPort: 2049, + }) + ); - efsFileSystem = new efs.FileSystem( - context.scope, this.options.name || `${id}-EfsFileSystem`, - { - vpc, - securityGroup: efsSG, - removalPolicy, - encrypted: true, - ...this.options.efsProps, - } - ); + efsFileSystem = new efs.FileSystem( + context.scope, + this.options.name || `${id}-EfsFileSystem`, + { + vpc, + securityGroup: efsSG, + removalPolicy, + ...this.options.efsProps, } - return efsFileSystem; + ); } + return efsFileSystem; + } } From dd788378052b6c15211f3bfccc70cc507ac646d0 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 10:18:26 -0600 Subject: [PATCH 06/33] Fixing the KMS key id confusion --- lib/resource-providers/kms-key.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/resource-providers/kms-key.ts b/lib/resource-providers/kms-key.ts index 97b52ae9b..022070cd9 100644 --- a/lib/resource-providers/kms-key.ts +++ b/lib/resource-providers/kms-key.ts @@ -30,16 +30,17 @@ export class KmsKeyProvider implements ResourceProvider { provide(context: ResourceContext): kms.IKey { const id = context.scope.node.id; + const keyId = `${id}-${this.aliasName ?? "default"}-KmsKey`; let key = undefined; if (this.aliasName) { - key = kms.Key.fromLookup(context.scope, `${id}-kms-key`, { + key = kms.Key.fromLookup(context.scope, keyId, { aliasName: this.aliasName, }); } if (!key) { - key = new kms.Key(context.scope, `${id}-kms-key`, { + key = new kms.Key(context.scope, keyId, { description: `Secrets Encryption Key for EKS Cluster '${context.blueprintProps.id}'`, ...this.kmsKeyProps, }); From 9abe304ad5cb0717645a820965ba12b0308a231b Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 10:56:27 -0600 Subject: [PATCH 07/33] Splitting kmskeyprovider to two: LookupKeyProvider and CreateKeyProvioder --- lib/resource-providers/kms-key.ts | 42 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/lib/resource-providers/kms-key.ts b/lib/resource-providers/kms-key.ts index 022070cd9..3dabb1d17 100644 --- a/lib/resource-providers/kms-key.ts +++ b/lib/resource-providers/kms-key.ts @@ -13,12 +13,12 @@ import { ResourceContext, ResourceProvider } from "../spi"; * .build(app, "east-test-1"); * ``` */ -export class KmsKeyProvider implements ResourceProvider { +export class CreateKmsKeyProvider implements ResourceProvider { private readonly aliasName?: string; private readonly kmsKeyProps?: kms.KeyProps; /** - * Pass either en aliasName to lookup an existing or pass optional key props to create a new one. + * Configuration options for the KMS Key. * * @param aliasName The alias name to lookup an existing KMS Key in the deployment target, if omitted a key will be created. * @param kmsKeyProps The key props used @@ -33,19 +33,33 @@ export class KmsKeyProvider implements ResourceProvider { const keyId = `${id}-${this.aliasName ?? "default"}-KmsKey`; let key = undefined; - if (this.aliasName) { - key = kms.Key.fromLookup(context.scope, keyId, { - aliasName: this.aliasName, - }); - } - - if (!key) { - key = new kms.Key(context.scope, keyId, { - description: `Secrets Encryption Key for EKS Cluster '${context.blueprintProps.id}'`, - ...this.kmsKeyProps, - }); - } + key = new kms.Key(context.scope, keyId, { + description: `Secrets Encryption Key for EKS Cluster '${context.blueprintProps.id}'`, + ...this.kmsKeyProps, + }); return key; } } + +/** + * Pass an aliasName to lookup an existing KMS Key. + * + * @param aliasName The alias name to lookup an existing KMS Key in the deployment target, if omitted a key will be created. + */ +export class LookupKmsKeyProvider implements ResourceProvider { + private readonly aliasName: string; + + public constructor(aliasName: string) { + this.aliasName = aliasName; + } + + provide(context: ResourceContext): kms.IKey { + const id = context.scope.node.id; + const keyId = `${id}-${this.aliasName}-KmsKey`; + + return kms.Key.fromLookup(context.scope, keyId, { + aliasName: this.aliasName, + }); + } +} From eca9f6090d17e416e66591488224e59c93988197 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 11:05:18 -0600 Subject: [PATCH 08/33] Updating the stack and tests --- lib/stacks/eks-blueprint-stack.ts | 711 +++++++++++++----------- test/resource-providers/kms-key.test.ts | 27 +- 2 files changed, 409 insertions(+), 329 deletions(-) diff --git a/lib/stacks/eks-blueprint-stack.ts b/lib/stacks/eks-blueprint-stack.ts index f2979ad92..3cbfb1d3c 100644 --- a/lib/stacks/eks-blueprint-stack.ts +++ b/lib/stacks/eks-blueprint-stack.ts @@ -1,99 +1,100 @@ -import * as cdk from 'aws-cdk-lib'; -import { IVpc } from 'aws-cdk-lib/aws-ec2'; -import { KubernetesVersion } from 'aws-cdk-lib/aws-eks'; -import { Construct } from 'constructs'; -import { MngClusterProvider } from '../cluster-providers/mng-cluster-provider'; -import { VpcProvider } from '../resource-providers/vpc'; -import * as spi from '../spi'; -import * as constraints from '../utils/constraints-utils'; -import * as utils from '../utils'; -import { cloneDeep } from '../utils'; +import * as cdk from "aws-cdk-lib"; +import { IVpc } from "aws-cdk-lib/aws-ec2"; +import { KubernetesVersion } from "aws-cdk-lib/aws-eks"; +import { Construct } from "constructs"; +import { MngClusterProvider } from "../cluster-providers/mng-cluster-provider"; +import { VpcProvider } from "../resource-providers/vpc"; +import * as spi from "../spi"; +import * as constraints from "../utils/constraints-utils"; +import * as utils from "../utils"; +import { cloneDeep } from "../utils"; import { IKey } from "aws-cdk-lib/aws-kms"; -import {KmsKeyProvider} from "../resource-providers/kms-key"; +import { CreateKmsKeyProvider } from "../resource-providers/kms-key"; import { ArgoGitOpsFactory } from "../addons/argocd/argo-gitops-factory"; export class EksBlueprintProps { - /** - * The id for the blueprint. - */ - readonly id: string; - - /** - * Defaults to id if not provided - */ - readonly name?: string; - - /** - * Add-ons if any. - */ - readonly addOns?: Array = []; - - /** - * Teams if any - */ - readonly teams?: Array = []; - - /** - * EC2 or Fargate are supported in the blueprint but any implementation conforming the interface - * will work - */ - readonly clusterProvider?: spi.ClusterProvider = new MngClusterProvider(); - - /** - * Kubernetes version (must be initialized for addons to work properly) - */ - readonly version?: KubernetesVersion = KubernetesVersion.V1_24; - - /** - * Named resource providers to leverage for cluster resources. - * The resource can represent Vpc, Hosting Zones or other resources, see {@link spi.ResourceType}. - * VPC for the cluster can be registered under the name of 'vpc' or as a single provider of type - */ - resourceProviders?: Map = new Map(); - - /** - * Control Plane log types to be enabled (if not passed, none) - * If wrong types are included, will throw an error. - */ - readonly enableControlPlaneLogTypes?: ControlPlaneLogType[]; - - /** - * If set to true and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), - * a default KMS encryption key will be used for envelope encryption of Kubernetes secrets (AWS managed new KMS key). - * If set to false, and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), then no secrets - * encyrption is applied. - * - * Default is true. - */ - readonly useDefaultSecretEncryption? : boolean = true; - - /** - * GitOps modes to be enabled. If not specified, GitOps mode is not enabled. - */ - readonly enableGitOpsMode?: spi.GitOpsMode; + /** + * The id for the blueprint. + */ + readonly id: string; + + /** + * Defaults to id if not provided + */ + readonly name?: string; + + /** + * Add-ons if any. + */ + readonly addOns?: Array = []; + + /** + * Teams if any + */ + readonly teams?: Array = []; + + /** + * EC2 or Fargate are supported in the blueprint but any implementation conforming the interface + * will work + */ + readonly clusterProvider?: spi.ClusterProvider = new MngClusterProvider(); + + /** + * Kubernetes version (must be initialized for addons to work properly) + */ + readonly version?: KubernetesVersion = KubernetesVersion.V1_24; + + /** + * Named resource providers to leverage for cluster resources. + * The resource can represent Vpc, Hosting Zones or other resources, see {@link spi.ResourceType}. + * VPC for the cluster can be registered under the name of 'vpc' or as a single provider of type + */ + resourceProviders?: Map = new Map(); + + /** + * Control Plane log types to be enabled (if not passed, none) + * If wrong types are included, will throw an error. + */ + readonly enableControlPlaneLogTypes?: ControlPlaneLogType[]; + + /** + * If set to true and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), + * a default KMS encryption key will be used for envelope encryption of Kubernetes secrets (AWS managed new KMS key). + * If set to false, and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), then no secrets + * encyrption is applied. + * + * Default is true. + */ + readonly useDefaultSecretEncryption?: boolean = true; + + /** + * GitOps modes to be enabled. If not specified, GitOps mode is not enabled. + */ + readonly enableGitOpsMode?: spi.GitOpsMode; } -export class BlueprintPropsConstraints implements constraints.ConstraintsType { - /** - * id can be no less than 1 character long, and no greater than 63 characters long. - * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ - */ - id = new constraints.StringConstraint(1, 63); - - /** - * name can be no less than 1 character long, and no greater than 63 characters long. - * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ - */ - name = new constraints.StringConstraint(1, 63); +export class BlueprintPropsConstraints + implements constraints.ConstraintsType +{ + /** + * id can be no less than 1 character long, and no greater than 63 characters long. + * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + */ + id = new constraints.StringConstraint(1, 63); + + /** + * name can be no less than 1 character long, and no greater than 63 characters long. + * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + */ + name = new constraints.StringConstraint(1, 63); } export const enum ControlPlaneLogType { - - API = 'api', - AUDIT = 'audit', - AUTHENTICATOR = 'authenticator', - CONTROLLER_MANAGER = 'controllerManager', - SCHEDULER = 'scheduler' + API = "api", + AUDIT = "audit", + AUTHENTICATOR = "authenticator", + CONTROLLER_MANAGER = "controllerManager", + SCHEDULER = "scheduler", } /** @@ -102,103 +103,134 @@ export const enum ControlPlaneLogType { * in accounts and regions. */ export class BlueprintBuilder implements spi.AsyncStackBuilder { - - props: Partial; - env: { - account?: string, - region?: string + props: Partial; + env: { + account?: string; + region?: string; + }; + + constructor() { + this.props = { + addOns: new Array(), + teams: new Array(), + resourceProviders: new Map(), }; - - constructor() { - this.props = { addOns: new Array(), teams: new Array(), resourceProviders: new Map() }; - this.env = { - account: process.env.CDK_DEFAULT_ACCOUNT, - region: process.env.CDK_DEFAULT_REGION - }; - } - - public name(name: string): this { - this.props = { ...this.props, ...{ name } }; - return this; - } - - public account(account?: string): this { - this.env.account = account; - return this; - } - - public region(region?: string): this { - this.env.region = region; - return this; - } - - public version(version: KubernetesVersion): this { - this.props = { ...this.props, ...{ version } }; - return this; - } - - public enableControlPlaneLogTypes(...types: ControlPlaneLogType[]): this { - this.props = { ...this.props, ...{ enableControlPlaneLogTypes: types } }; - return this; - } - - public enableGitOps(mode?: spi.GitOpsMode): this { - this.props = { ...this.props, ...{ enableGitOpsMode: mode ?? spi.GitOpsMode.APP_OF_APPS } }; - return this; - } - - public withBlueprintProps(props: Partial): this { - const resourceProviders = this.props.resourceProviders!; - this.props = { ...this.props, ...cloneDeep(props) }; - if (props.resourceProviders) { - this.props.resourceProviders = new Map([...resourceProviders!.entries(), ...props.resourceProviders.entries()]); - } - return this; - } - - public addOns(...addOns: spi.ClusterAddOn[]): this { - this.props = { ...this.props, ...{ addOns: this.props.addOns?.concat(addOns) } }; - return this; - } - - public clusterProvider(clusterProvider: spi.ClusterProvider) { - this.props = { ...this.props, ...{ clusterProvider: clusterProvider } }; - return this; - } - - public id(id: string): this { - this.props = { ...this.props, ...{ id } }; - return this; - } - - public teams(...teams: spi.Team[]): this { - this.props = { ...this.props, ...{ teams: this.props.teams?.concat(teams) } }; - return this; - } - - public resourceProvider(name: string, provider: spi.ResourceProvider): this { - this.props.resourceProviders?.set(name, provider); - return this; - } - - public useDefaultSecretEncryption(useDefault: boolean): this { - this.props = { ...this.props, ...{ useDefaultSecretEncryption: useDefault } }; - return this; - } - - public clone(region?: string, account?: string): BlueprintBuilder { - return new BlueprintBuilder().withBlueprintProps({ ...this.props }) - .account(account ?? this.env.account).region(region ?? this.env.region); - } - - public build(scope: Construct, id: string, stackProps?: cdk.StackProps): EksBlueprint { - return new EksBlueprint(scope, { ...this.props, ...{ id } }, - { ...{ env: this.env }, ...stackProps }); + this.env = { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }; + } + + public name(name: string): this { + this.props = { ...this.props, ...{ name } }; + return this; + } + + public account(account?: string): this { + this.env.account = account; + return this; + } + + public region(region?: string): this { + this.env.region = region; + return this; + } + + public version(version: KubernetesVersion): this { + this.props = { ...this.props, ...{ version } }; + return this; + } + + public enableControlPlaneLogTypes(...types: ControlPlaneLogType[]): this { + this.props = { ...this.props, ...{ enableControlPlaneLogTypes: types } }; + return this; + } + + public enableGitOps(mode?: spi.GitOpsMode): this { + this.props = { + ...this.props, + ...{ enableGitOpsMode: mode ?? spi.GitOpsMode.APP_OF_APPS }, + }; + return this; + } + + public withBlueprintProps(props: Partial): this { + const resourceProviders = this.props.resourceProviders!; + this.props = { ...this.props, ...cloneDeep(props) }; + if (props.resourceProviders) { + this.props.resourceProviders = new Map([ + ...resourceProviders!.entries(), + ...props.resourceProviders.entries(), + ]); } + return this; + } - public async buildAsync(scope: Construct, id: string, stackProps?: cdk.StackProps): Promise { - return this.build(scope, id, stackProps).waitForAsyncTasks(); - } + public addOns(...addOns: spi.ClusterAddOn[]): this { + this.props = { + ...this.props, + ...{ addOns: this.props.addOns?.concat(addOns) }, + }; + return this; + } + + public clusterProvider(clusterProvider: spi.ClusterProvider) { + this.props = { ...this.props, ...{ clusterProvider: clusterProvider } }; + return this; + } + + public id(id: string): this { + this.props = { ...this.props, ...{ id } }; + return this; + } + + public teams(...teams: spi.Team[]): this { + this.props = { + ...this.props, + ...{ teams: this.props.teams?.concat(teams) }, + }; + return this; + } + + public resourceProvider(name: string, provider: spi.ResourceProvider): this { + this.props.resourceProviders?.set(name, provider); + return this; + } + + public useDefaultSecretEncryption(useDefault: boolean): this { + this.props = { + ...this.props, + ...{ useDefaultSecretEncryption: useDefault }, + }; + return this; + } + + public clone(region?: string, account?: string): BlueprintBuilder { + return new BlueprintBuilder() + .withBlueprintProps({ ...this.props }) + .account(account ?? this.env.account) + .region(region ?? this.env.region); + } + + public build( + scope: Construct, + id: string, + stackProps?: cdk.StackProps + ): EksBlueprint { + return new EksBlueprint( + scope, + { ...this.props, ...{ id } }, + { ...{ env: this.env }, ...stackProps } + ); + } + + public async buildAsync( + scope: Construct, + id: string, + stackProps?: cdk.StackProps + ): Promise { + return this.build(scope, id, stackProps).waitForAsyncTasks(); + } } /** @@ -206,156 +238,201 @@ export class BlueprintBuilder implements spi.AsyncStackBuilder { * and orchestrates provisioning of add-ons, teams and post deployment hooks. */ export class EksBlueprint extends cdk.Stack { - - static readonly USAGE_ID = "qs-1s1r465hk"; - - private asyncTasks: Promise; - - private clusterInfo: spi.ClusterInfo; - - public static builder(): BlueprintBuilder { - return new BlueprintBuilder(); + static readonly USAGE_ID = "qs-1s1r465hk"; + + private asyncTasks: Promise; + + private clusterInfo: spi.ClusterInfo; + + public static builder(): BlueprintBuilder { + return new BlueprintBuilder(); + } + + constructor( + scope: Construct, + blueprintProps: EksBlueprintProps, + props?: cdk.StackProps + ) { + super( + scope, + blueprintProps.id, + utils.withUsageTracking(EksBlueprint.USAGE_ID, props) + ); + this.validateInput(blueprintProps); + + const resourceContext = this.provideNamedResources(blueprintProps); + + let vpcResource: IVpc | undefined = resourceContext.get( + spi.GlobalResources.Vpc + ); + + if (!vpcResource) { + vpcResource = resourceContext.add( + spi.GlobalResources.Vpc, + new VpcProvider() + ); } - constructor(scope: Construct, blueprintProps: EksBlueprintProps, props?: cdk.StackProps) { - super(scope, blueprintProps.id, utils.withUsageTracking(EksBlueprint.USAGE_ID, props)); - this.validateInput(blueprintProps); - - const resourceContext = this.provideNamedResources(blueprintProps); + const version = blueprintProps.version ?? KubernetesVersion.V1_24; + let kmsKeyResource: IKey | undefined = resourceContext.get( + spi.GlobalResources.KmsKey + ); - let vpcResource: IVpc | undefined = resourceContext.get(spi.GlobalResources.Vpc); - - if (!vpcResource) { - vpcResource = resourceContext.add(spi.GlobalResources.Vpc, new VpcProvider()); - } - - const version = blueprintProps.version ?? KubernetesVersion.V1_24; - let kmsKeyResource: IKey | undefined = resourceContext.get(spi.GlobalResources.KmsKey); - - if (!kmsKeyResource && blueprintProps.useDefaultSecretEncryption != false) { - kmsKeyResource = resourceContext.add(spi.GlobalResources.KmsKey, new KmsKeyProvider()); - } - - blueprintProps = this.resolveDynamicProxies(blueprintProps, resourceContext); - - const clusterProvider = blueprintProps.clusterProvider ?? new MngClusterProvider({ - id: `${blueprintProps.name ?? blueprintProps.id}-ng`, - version - }); - - this.clusterInfo = clusterProvider.createCluster(this, vpcResource!, kmsKeyResource); - this.clusterInfo.setResourceContext(resourceContext); - - let enableLogTypes: string[] | undefined = blueprintProps.enableControlPlaneLogTypes; - if (enableLogTypes) { - utils.setupClusterLogging(this.clusterInfo.cluster.stack, this.clusterInfo.cluster, enableLogTypes); - } - - if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APPLICATION) { - ArgoGitOpsFactory.enableGitOps(); - } else if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APP_OF_APPS) { - ArgoGitOpsFactory.enableGitOpsAppOfApps(); - } - - const postDeploymentSteps = Array(); - - for (let addOn of (blueprintProps.addOns ?? [])) { // must iterate in the strict order - const result = addOn.deploy(this.clusterInfo); - if (result) { - const addOnKey = utils.getAddOnNameOrId(addOn); - this.clusterInfo.addScheduledAddOn(addOnKey, result, utils.isOrderedAddOn(addOn)); - } - const postDeploy: any = addOn; - if ((postDeploy as spi.ClusterPostDeploy).postDeploy !== undefined) { - postDeploymentSteps.push(postDeploy); - } - } + if (!kmsKeyResource && blueprintProps.useDefaultSecretEncryption != false) { + kmsKeyResource = resourceContext.add( + spi.GlobalResources.KmsKey, + new CreateKmsKeyProvider() + ); + } - const scheduledAddOns = this.clusterInfo.getAllScheduledAddons(); - const addOnKeys = [...scheduledAddOns.keys()]; - const promises = scheduledAddOns.values(); - - this.asyncTasks = Promise.all(promises).then((constructs) => { - constructs.forEach((construct, index) => { - this.clusterInfo.addProvisionedAddOn(addOnKeys[index], construct); - }); - - if (blueprintProps.teams != null) { - for (let team of blueprintProps.teams) { - team.setup(this.clusterInfo); - } - } - - for (let step of postDeploymentSteps) { - step.postDeploy(this.clusterInfo, blueprintProps.teams ?? []); - } - }); - - this.asyncTasks.catch(err => { - console.error(err); - throw new Error(err); - }); + blueprintProps = this.resolveDynamicProxies( + blueprintProps, + resourceContext + ); + + const clusterProvider = + blueprintProps.clusterProvider ?? + new MngClusterProvider({ + id: `${blueprintProps.name ?? blueprintProps.id}-ng`, + version, + }); + + this.clusterInfo = clusterProvider.createCluster( + this, + vpcResource!, + kmsKeyResource + ); + this.clusterInfo.setResourceContext(resourceContext); + + let enableLogTypes: string[] | undefined = + blueprintProps.enableControlPlaneLogTypes; + if (enableLogTypes) { + utils.setupClusterLogging( + this.clusterInfo.cluster.stack, + this.clusterInfo.cluster, + enableLogTypes + ); } - /** - * Since constructor cannot be marked as async, adding a separate method to wait - * for async code to finish. - * @returns Promise that resolves to the blueprint - */ - public async waitForAsyncTasks(): Promise { - if (this.asyncTasks) { - return this.asyncTasks.then(() => { - return this; - }); - } - return Promise.resolve(this); + if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APPLICATION) { + ArgoGitOpsFactory.enableGitOps(); + } else if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APP_OF_APPS) { + ArgoGitOpsFactory.enableGitOpsAppOfApps(); } - /** - * This method returns all the constructs produced by during the cluster creation (e.g. add-ons). - * May be used in testing for verification. - * @returns cluster info object - */ - getClusterInfo(): spi.ClusterInfo { - return this.clusterInfo; + const postDeploymentSteps = Array(); + + for (let addOn of blueprintProps.addOns ?? []) { + // must iterate in the strict order + const result = addOn.deploy(this.clusterInfo); + if (result) { + const addOnKey = utils.getAddOnNameOrId(addOn); + this.clusterInfo.addScheduledAddOn( + addOnKey, + result, + utils.isOrderedAddOn(addOn) + ); + } + const postDeploy: any = addOn; + if ((postDeploy as spi.ClusterPostDeploy).postDeploy !== undefined) { + postDeploymentSteps.push(postDeploy); + } } - private provideNamedResources(blueprintProps: EksBlueprintProps): spi.ResourceContext { - const result = new spi.ResourceContext(this, blueprintProps); + const scheduledAddOns = this.clusterInfo.getAllScheduledAddons(); + const addOnKeys = [...scheduledAddOns.keys()]; + const promises = scheduledAddOns.values(); - for (let [key, value] of blueprintProps.resourceProviders ?? []) { - result.add(key, value); - } + this.asyncTasks = Promise.all(promises).then((constructs) => { + constructs.forEach((construct, index) => { + this.clusterInfo.addProvisionedAddOn(addOnKeys[index], construct); + }); - return result; + if (blueprintProps.teams != null) { + for (let team of blueprintProps.teams) { + team.setup(this.clusterInfo); + } + } + + for (let step of postDeploymentSteps) { + step.postDeploy(this.clusterInfo, blueprintProps.teams ?? []); + } + }); + + this.asyncTasks.catch((err) => { + console.error(err); + throw new Error(err); + }); + } + + /** + * Since constructor cannot be marked as async, adding a separate method to wait + * for async code to finish. + * @returns Promise that resolves to the blueprint + */ + public async waitForAsyncTasks(): Promise { + if (this.asyncTasks) { + return this.asyncTasks.then(() => { + return this; + }); } - - /** - * Resolves all dynamic proxies, that substitutes resource provider proxies with the resolved values. - * @param blueprintProps - * @param resourceContext - * @returns a copy of blueprint props with resolved values - */ - private resolveDynamicProxies(blueprintProps: EksBlueprintProps, resourceContext: spi.ResourceContext) : EksBlueprintProps { - return utils.cloneDeep(blueprintProps, (value) => { - return utils.resolveTarget(value, resourceContext); - }); + return Promise.resolve(this); + } + + /** + * This method returns all the constructs produced by during the cluster creation (e.g. add-ons). + * May be used in testing for verification. + * @returns cluster info object + */ + getClusterInfo(): spi.ClusterInfo { + return this.clusterInfo; + } + + private provideNamedResources( + blueprintProps: EksBlueprintProps + ): spi.ResourceContext { + const result = new spi.ResourceContext(this, blueprintProps); + + for (let [key, value] of blueprintProps.resourceProviders ?? []) { + result.add(key, value); } - /** - * Validates input against basic defined constraints. - * @param blueprintProps - */ - private validateInput(blueprintProps: EksBlueprintProps) { - const teamNames = new Set(); - constraints.validateConstraints(new BlueprintPropsConstraints, EksBlueprintProps.name, blueprintProps); - if (blueprintProps.teams) { - blueprintProps.teams.forEach(e => { - if (teamNames.has(e.name)) { - throw new Error(`Team ${e.name} is registered more than once`); - } - teamNames.add(e.name); - }); + return result; + } + + /** + * Resolves all dynamic proxies, that substitutes resource provider proxies with the resolved values. + * @param blueprintProps + * @param resourceContext + * @returns a copy of blueprint props with resolved values + */ + private resolveDynamicProxies( + blueprintProps: EksBlueprintProps, + resourceContext: spi.ResourceContext + ): EksBlueprintProps { + return utils.cloneDeep(blueprintProps, (value) => { + return utils.resolveTarget(value, resourceContext); + }); + } + + /** + * Validates input against basic defined constraints. + * @param blueprintProps + */ + private validateInput(blueprintProps: EksBlueprintProps) { + const teamNames = new Set(); + constraints.validateConstraints( + new BlueprintPropsConstraints(), + EksBlueprintProps.name, + blueprintProps + ); + if (blueprintProps.teams) { + blueprintProps.teams.forEach((e) => { + if (teamNames.has(e.name)) { + throw new Error(`Team ${e.name} is registered more than once`); } + teamNames.add(e.name); + }); } -} \ No newline at end of file + } +} diff --git a/test/resource-providers/kms-key.test.ts b/test/resource-providers/kms-key.test.ts index 41329c805..9cb80c763 100644 --- a/test/resource-providers/kms-key.test.ts +++ b/test/resource-providers/kms-key.test.ts @@ -1,7 +1,7 @@ import { App } from "aws-cdk-lib"; import * as blueprints from "../../lib"; import { Match, Template } from "aws-cdk-lib/assertions"; -import { KmsKeyProvider } from "../../lib/resource-providers/kms-key"; +import { CreateKmsKeyProvider } from "../../lib/resource-providers/kms-key"; import { GlobalResources } from "../../lib"; describe("KmsKeyProvider", () => { @@ -46,7 +46,7 @@ describe("KmsKeyProvider", () => { const stack = blueprints.EksBlueprint.builder() .resourceProvider( GlobalResources.KmsKey, - new KmsKeyProvider("my-custom-eks-key") + new CreateKmsKeyProvider("my-custom-eks-key") ) .account("123456789012") .region("us-east-1") @@ -78,7 +78,7 @@ describe("KmsKeyProvider", () => { const stack = blueprints.EksBlueprint.builder() .resourceProvider( GlobalResources.KmsKey, - new KmsKeyProvider(undefined, { alias: "any-alias" }) + new CreateKmsKeyProvider(undefined, { alias: "any-alias" }) ) .account("123456789012") .region("us-east-1") @@ -125,18 +125,21 @@ describe("KmsKeyProvider", () => { const template = Template.fromStack(stack); // Then EKS cluster config has no encryption - template.resourcePropertiesCountIs("Custom::AWSCDK-EKS-Cluster", { + template.resourcePropertiesCountIs( + "Custom::AWSCDK-EKS-Cluster", + { Config: { - encryptionConfig: [ + encryptionConfig: [ { - provider: { - keyArn: Match.anyValue(), - }, - resources: ["secrets"], + provider: { + keyArn: Match.anyValue(), + }, + resources: ["secrets"], }, - ], + ], }, - }, - 0); + }, + 0 + ); }); }); From 0c9224390880aef6194637201dea5e1a2603781a Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 11:23:28 -0600 Subject: [PATCH 09/33] Splitting EfsFileSystemProvider into two CreateEfsFileSystemProvider and LookupEfsFileSystemProvider --- lib/resource-providers/efs.ts | 129 ++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 53 deletions(-) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 785717085..7355d4798 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -3,31 +3,30 @@ import * as efs from "aws-cdk-lib/aws-efs"; import * as cdk from "aws-cdk-lib/core"; import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; -/** - * Configuration options for the EFS file system. - */ -export interface EfsFileSystemProps { +export interface CreateEfsFileSystemProps { readonly name?: string; - readonly fileSystemId?: string; readonly efsProps?: Omit; readonly removalPolicy?: cdk.RemovalPolicy; } +export interface LookupEfsFileSystemProps { + readonly name: string; + readonly fileSystemId: string; +} + /** * EFS resource provider. * - * Pass either an EFS file system name and id to lookup an existing one, or create a new one. * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. - * @param fileSystemId The id of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. * @param efsProps The props used for the file system. * @param removalPolicy The removal policy to use for the EFS file system. */ -export class EfsFileSystemProvider +export class CreateEfsFileSystemProvider implements ResourceProvider { - readonly options: EfsFileSystemProps; + readonly options: CreateEfsFileSystemProps; - constructor(options: EfsFileSystemProps) { + constructor(options: CreateEfsFileSystemProps) { this.options = options; } @@ -38,54 +37,78 @@ export class EfsFileSystemProvider }-EfsSecurityGroup`; let efsFileSystem: efs.IFileSystem | undefined; - if (this.options.fileSystemId && this.options.name) { - const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( - context.scope, - securityGroupId, - securityGroupId - ); - efsFileSystem = efs.FileSystem.fromFileSystemAttributes( - context.scope, - this.options.name, - { - securityGroup: securityGroup, - fileSystemId: this.options.fileSystemId, - } - ); + const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; + if (vpc === undefined) { + throw new Error("VPC not found in context"); } + const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; + const clusterVpcCidr = vpc.vpcCidrBlock; - if (!efsFileSystem) { - const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; - if (vpc === undefined) { - throw new Error("VPC not found in context"); - } - const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; - const clusterVpcCidr = vpc.vpcCidrBlock; + const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { + vpc, + securityGroupName: securityGroupId, + }); + efsSG.addIngressRule( + ec2.Peer.ipv4(clusterVpcCidr), + new ec2.Port({ + protocol: ec2.Protocol.TCP, + stringRepresentation: "EFSconnection", + toPort: 2049, + fromPort: 2049, + }) + ); - const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { + efsFileSystem = new efs.FileSystem( + context.scope, + this.options.name || `${id}-EfsFileSystem`, + { vpc, - securityGroupName: securityGroupId, - }); - efsSG.addIngressRule( - ec2.Peer.ipv4(clusterVpcCidr), - new ec2.Port({ - protocol: ec2.Protocol.TCP, - stringRepresentation: "EFSconnection", - toPort: 2049, - fromPort: 2049, - }) - ); + securityGroup: efsSG, + removalPolicy, + ...this.options.efsProps, + } + ); + return efsFileSystem; + } +} - efsFileSystem = new efs.FileSystem( - context.scope, - this.options.name || `${id}-EfsFileSystem`, - { - vpc, - securityGroup: efsSG, - removalPolicy, - ...this.options.efsProps, - } - ); +/** + * Pass an EFS file system name and id to lookup an existing EFS file system. + * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + * @param fileSystemId The id of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + */ +export class LookupEfsFileSystemProvider + implements ResourceProvider +{ + readonly options: LookupEfsFileSystemProps; + + constructor(options: LookupEfsFileSystemProps) { + this.options = options; + } + + provide(context: ResourceContext): efs.IFileSystem { + const id = context.scope.node.id; + const securityGroupId = `${id}-${ + this.options.name ?? "default" + }-EfsSecurityGroup`; + let efsFileSystem: efs.IFileSystem | undefined; + + const securityGroup = ec2.SecurityGroup.fromSecurityGroupId( + context.scope, + securityGroupId, + securityGroupId + ); + efsFileSystem = efs.FileSystem.fromFileSystemAttributes( + context.scope, + this.options.name, + { + securityGroup: securityGroup, + fileSystemId: this.options.fileSystemId, + } + ); + + if (!efsFileSystem) { + throw new Error("EFS file system not found"); } return efsFileSystem; } From 8f4a6bbdd323fb533f2f8046fb07ed7279059748 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 12:49:35 -0600 Subject: [PATCH 10/33] Using the KMS key from the context --- lib/resource-providers/efs.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 7355d4798..7bb0a1d2b 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -1,11 +1,13 @@ import * as ec2 from "aws-cdk-lib/aws-ec2"; import * as efs from "aws-cdk-lib/aws-efs"; +import * as kms from "aws-cdk-lib/aws-kms"; import * as cdk from "aws-cdk-lib/core"; import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; export interface CreateEfsFileSystemProps { readonly name?: string; - readonly efsProps?: Omit; + readonly efsProps?: Omit; + readonly kmsKeyResourceName?: string; readonly removalPolicy?: cdk.RemovalPolicy; } @@ -36,13 +38,16 @@ export class CreateEfsFileSystemProvider this.options.name ?? "default" }-EfsSecurityGroup`; let efsFileSystem: efs.IFileSystem | undefined; - const vpc = context.get(GlobalResources.Vpc) as ec2.IVpc; if (vpc === undefined) { throw new Error("VPC not found in context"); } const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; const clusterVpcCidr = vpc.vpcCidrBlock; + let kmsKey: kms.IKey | undefined; + if (this.options.kmsKeyResourceName) { + kmsKey = context.get(this.options.kmsKeyResourceName) as kms.IKey; + } const efsSG = new ec2.SecurityGroup(context.scope, securityGroupId, { vpc, @@ -65,6 +70,7 @@ export class CreateEfsFileSystemProvider vpc, securityGroup: efsSG, removalPolicy, + kmsKey, ...this.options.efsProps, } ); From ca1d690257fb67e9b37ec462fb8002b82f3af71b Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 12:53:38 -0600 Subject: [PATCH 11/33] Fixing the KMS key props --- lib/resource-providers/kms-key.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/resource-providers/kms-key.ts b/lib/resource-providers/kms-key.ts index 3dabb1d17..42df03116 100644 --- a/lib/resource-providers/kms-key.ts +++ b/lib/resource-providers/kms-key.ts @@ -34,7 +34,8 @@ export class CreateKmsKeyProvider implements ResourceProvider { let key = undefined; key = new kms.Key(context.scope, keyId, { - description: `Secrets Encryption Key for EKS Cluster '${context.blueprintProps.id}'`, + alias: this.aliasName, + description: `Key for EKS Cluster '${context.blueprintProps.id}'`, ...this.kmsKeyProps, }); From ce03a2142d30322741a8aca23a5dcbe6aed6607d Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 8 Mar 2023 14:03:26 -0600 Subject: [PATCH 12/33] Addressing AWS Load Balancer dependency issue --- lib/addons/jupyterhub/index.ts | 4 +++- test/jupyterhub.test.ts | 29 ++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/addons/jupyterhub/index.ts b/lib/addons/jupyterhub/index.ts index 7a9ec0732..361cb9b7b 100644 --- a/lib/addons/jupyterhub/index.ts +++ b/lib/addons/jupyterhub/index.ts @@ -135,7 +135,6 @@ export class JupyterHubAddOn extends HelmAddOn { this.options = this.props as JupyterHubAddOnProps; } - @dependable(AwsLoadBalancerControllerAddOn.name) deploy(clusterInfo: ClusterInfo): Promise { const cluster = clusterInfo.cluster; let values = this.options.values ?? {}; @@ -202,8 +201,10 @@ export class JupyterHubAddOn extends HelmAddOn { const ingressAnnotations = this.options.ingressAnnotations; const cert = this.options.certificateResourceName; + const albAddOnCheck = clusterInfo.getScheduledAddOn('AwsLoadBalancerControllerAddOn.name'); // Use Ingress and AWS ALB if (serviceType == JupyterHubServiceType.ALB){ + assert(albAddOnCheck, `Missing a dependency: ${AwsLoadBalancerControllerAddOn.name}. Please add it to your list of addons.`); const presetAnnotations: any = { 'alb.ingress.kubernetes.io/scheme': 'internet-facing', 'alb.ingress.kubernetes.io/target-type': 'ip', @@ -229,6 +230,7 @@ export class JupyterHubAddOn extends HelmAddOn { setPath(values, "proxy.service", {"type": "ClusterIP"}); // We will use NLB } else { + assert(albAddOnCheck, `Missing a dependency: ${AwsLoadBalancerControllerAddOn.name}. Please add it to your list of addons.`); setPath(values, "proxy.service", { "annotations": { "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", diff --git a/test/jupyterhub.test.ts b/test/jupyterhub.test.ts index 2659376bb..ee00ca9e4 100644 --- a/test/jupyterhub.test.ts +++ b/test/jupyterhub.test.ts @@ -49,7 +49,7 @@ describe('Unit tests for JupyterHub addon', () => { }).toThrow("Missing a dependency: EfsCsiDriverAddOn. Please add it to your list of addons."); }); - test("Stack creation fails due to no AWS Load Balancer Controller add-on", () => { + test("Stack creation fails due to no AWS Load Balancer Controller add-on when using ALB", () => { const app = new cdk.App(); const blueprint = blueprints.EksBlueprint.builder(); @@ -68,8 +68,31 @@ describe('Unit tests for JupyterHub addon', () => { .teams(new blueprints.PlatformTeam({ name: 'platform' })); expect(()=> { - blueprint.build(app, 'stack-with-no-efs-csi-addon'); - }).toThrow("Missing a dependency for AwsLoadBalancerControllerAddOn for stack-with-no-efs-csi-addon"); + blueprint.build(app, 'stack-with-no-aws-load-balancer-controller-addon'); + }).toThrow("Missing a dependency: AwsLoadBalancerControllerAddOn. Please add it to your list of addons."); + }); + + test("Stack creation fails due to no AWS Load Balancer Controller add-on when using NLB", () => { + const app = new cdk.App(); + + const blueprint = blueprints.EksBlueprint.builder(); + + blueprint.account("123567891").region('us-west-1') + .addOns( + new blueprints.EfsCsiDriverAddOn, + new blueprints.JupyterHubAddOn({ + efsConfig: { + pvcName: "efs-persist", + removalPolicy: cdk.RemovalPolicy.DESTROY, + capacity: '100Gi', + }, + serviceType: JupyterHubServiceType.NLB, + })) + .teams(new blueprints.PlatformTeam({ name: 'platform' })); + + expect(()=> { + blueprint.build(app, 'stack-with-no-aws-load-balancer-controller-addon'); + }).toThrow("Missing a dependency: AwsLoadBalancerControllerAddOn. Please add it to your list of addons."); }); }); From 087ee89cf8ec26b1620696896373bb41274d3d4b Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Wed, 8 Mar 2023 15:34:49 -0600 Subject: [PATCH 13/33] Adding KMS keys to the example blueprint --- examples/blueprint-construct/index.ts | 542 ++++++++++++++------------ 1 file changed, 292 insertions(+), 250 deletions(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 5b15bd019..d8a272be9 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -1,268 +1,310 @@ -import * as cdk from 'aws-cdk-lib'; +import * as cdk from "aws-cdk-lib"; import * as ec2 from "aws-cdk-lib/aws-ec2"; -import { CapacityType, KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; -import * as kms from 'aws-cdk-lib/aws-kms'; -import { AccountRootPrincipal, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; +import { + CapacityType, + KubernetesVersion, + NodegroupAmiType, +} from "aws-cdk-lib/aws-eks"; +import { + AccountRootPrincipal, + PolicyStatement, + Role, +} from "aws-cdk-lib/aws-iam"; +import * as kms from "aws-cdk-lib/aws-kms"; import { Construct } from "constructs"; -import * as blueprints from '../../lib'; -import { logger, userLog } from '../../lib/utils'; -import * as team from '../teams'; -import { VpcProvider } from '../../lib'; +import * as blueprints from "../../lib"; +import { VpcProvider } from "../../lib"; +import { logger, userLog } from "../../lib/utils"; +import * as team from "../teams"; -const burnhamManifestDir = './examples/teams/team-burnham/'; -const rikerManifestDir = './examples/teams/team-riker/'; +const burnhamManifestDir = "./examples/teams/team-burnham/"; +const rikerManifestDir = "./examples/teams/team-riker/"; const teamManifestDirList = [burnhamManifestDir, rikerManifestDir]; export interface BlueprintConstructProps { - /** - * Id - */ - id: string + /** + * Id + */ + id: string; } export default class BlueprintConstruct { - constructor(scope: Construct, props: cdk.StackProps) { + constructor(scope: Construct, props: cdk.StackProps) { + blueprints.HelmAddOn.validateHelmVersions = true; + blueprints.HelmAddOn.failOnVersionValidation = false; + logger.settings.minLevel = 3; + userLog.settings.minLevel = 2; - blueprints.HelmAddOn.validateHelmVersions = true; - blueprints.HelmAddOn.failOnVersionValidation = false; - logger.settings.minLevel = 3; - userLog.settings.minLevel = 2; + // TODO: fix IAM user provisioning for admin user + // Setup platform team. + //const account = props.env!.account! + // const platformTeam = new team.TeamPlatform(account) + // Teams for the cluster. + const teams: Array = [ + new team.TeamTroi(), + new team.TeamRiker(scope, teamManifestDirList[1]), + new team.TeamBurnham(scope, teamManifestDirList[0]), + new team.TeamPlatform(process.env.CDK_DEFAULT_ACCOUNT!), + ]; + const prodBootstrapArgo = new blueprints.addons.ArgoCDAddOn({ + // TODO: enabling this cause stack deletion failure, known issue: + // https://github.com/aws-quickstart/cdk-eks-blueprints/blob/main/docs/addons/argo-cd.md#known-issues + // bootstrapRepo: { + // repoUrl: 'https://github.com/aws-samples/eks-blueprints-add-ons.git', + // path: 'chart', + // targetRevision: "eks-blueprints-cdk", + // }, + // workloadApplications: [ + // { + // name: "micro-services", + // namespace: "argocd", + // repository: { + // repoUrl: 'https://github.com/aws-samples/eks-blueprints-workloads.git', + // path: 'envs/dev', + // targetRevision: "main", + // }, + // values: { + // domain: "" + // } + // } + // ], + // adminPasswordSecretName: "argo-admin-secret" + }); + const addOns: Array = [ + new blueprints.addons.AppMeshAddOn(), + new blueprints.addons.CertManagerAddOn(), + new blueprints.addons.KubeStateMetricsAddOn(), + new blueprints.addons.PrometheusNodeExporterAddOn(), + new blueprints.addons.AdotCollectorAddOn(), + new blueprints.addons.AmpAddOn(), + new blueprints.addons.XrayAdotAddOn(), + // new blueprints.addons.CloudWatchAdotAddOn(), + new blueprints.addons.IstioBaseAddOn(), + new blueprints.addons.IstioControlPlaneAddOn(), + new blueprints.addons.CalicoOperatorAddOn(), + new blueprints.addons.MetricsServerAddOn(), + new blueprints.addons.AwsLoadBalancerControllerAddOn(), + new blueprints.addons.SecretsStoreAddOn(), + prodBootstrapArgo, + new blueprints.addons.SSMAgentAddOn(), + new blueprints.addons.NginxAddOn({ + values: { + controller: { service: { create: false } }, + }, + }), + new blueprints.addons.VeleroAddOn(), + new blueprints.addons.VpcCniAddOn({ + customNetworkingConfig: { + subnets: [ + blueprints.getNamedResource("secondary-cidr-subnet-0"), + blueprints.getNamedResource("secondary-cidr-subnet-1"), + blueprints.getNamedResource("secondary-cidr-subnet-2"), + ], + }, + awsVpcK8sCniCustomNetworkCfg: true, + eniConfigLabelDef: "topology.kubernetes.io/zone", + }), + new blueprints.addons.CoreDnsAddOn(), + new blueprints.addons.KubeProxyAddOn(), + new blueprints.addons.OpaGatekeeperAddOn(), + new blueprints.addons.AckAddOn({ + id: "s3-ack", + createNamespace: true, + skipVersionValidation: true, + serviceName: blueprints.AckServiceName.S3, + }), + new blueprints.addons.KarpenterAddOn({ + requirements: [ + { + key: "node.kubernetes.io/instance-type", + op: "In", + vals: ["m5.2xlarge"], + }, + { + key: "topology.kubernetes.io/zone", + op: "NotIn", + vals: ["us-west-2c"], + }, + { key: "kubernetes.io/arch", op: "In", vals: ["amd64", "arm64"] }, + { key: "karpenter.sh/capacity-type", op: "In", vals: ["spot"] }, + ], + subnetTags: { + Name: "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", + }, + securityGroupTags: { + "kubernetes.io/cluster/blueprint-construct-dev": "owned", + }, + taints: [ + { + key: "workload", + value: "test", + effect: "NoSchedule", + }, + ], + consolidation: { enabled: true }, + ttlSecondsUntilExpired: 2592000, + weight: 20, + interruptionHandling: true, + limits: { + resources: { + cpu: 20, + memory: "64Gi", + }, + }, + }), + new blueprints.addons.AwsNodeTerminationHandlerAddOn(), + new blueprints.addons.KubeviousAddOn(), + new blueprints.addons.EbsCsiDriverAddOn({ + kmsKeys: [ + blueprints.getResource( + (context) => + new kms.Key(context.scope, "ebs-csi-driver-key", { + alias: "ebs-csi-driver-key", + }) + ), + ], + }), + new blueprints.addons.EfsCsiDriverAddOn({ + replicaCount: 1, + kmsKeys: [ + blueprints.getResource( + (context) => + new kms.Key(context.scope, "efs-csi-driver-key", { + alias: "efs-csi-driver-key", + }) + ), + ], + }), + new blueprints.addons.KedaAddOn({ + podSecurityContextFsGroup: 1001, + securityContextRunAsGroup: 1001, + securityContextRunAsUser: 1001, + irsaRoles: ["CloudWatchFullAccess", "AmazonSQSFullAccess"], + }), + new blueprints.addons.AWSPrivateCAIssuerAddon(), + new blueprints.addons.JupyterHubAddOn({ + efsConfig: { + pvcName: "efs-persist", + removalPolicy: cdk.RemovalPolicy.DESTROY, + capacity: "10Gi", + }, + serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, + notebookStack: "jupyter/datascience-notebook", + values: { prePuller: { hook: { enabled: false } } }, + }), + new blueprints.EmrEksAddOn(), + new blueprints.AwsBatchAddOn(), + ]; - // TODO: fix IAM user provisioning for admin user - // Setup platform team. - //const account = props.env!.account! - // const platformTeam = new team.TeamPlatform(account) - // Teams for the cluster. - const teams: Array = [ - new team.TeamTroi, - new team.TeamRiker(scope, teamManifestDirList[1]), - new team.TeamBurnham(scope, teamManifestDirList[0]), - new team.TeamPlatform(process.env.CDK_DEFAULT_ACCOUNT!) - ]; - const prodBootstrapArgo = new blueprints.addons.ArgoCDAddOn({ - // TODO: enabling this cause stack deletion failure, known issue: - // https://github.com/aws-quickstart/cdk-eks-blueprints/blob/main/docs/addons/argo-cd.md#known-issues - // bootstrapRepo: { - // repoUrl: 'https://github.com/aws-samples/eks-blueprints-add-ons.git', - // path: 'chart', - // targetRevision: "eks-blueprints-cdk", - // }, - // workloadApplications: [ - // { - // name: "micro-services", - // namespace: "argocd", - // repository: { - // repoUrl: 'https://github.com/aws-samples/eks-blueprints-workloads.git', - // path: 'envs/dev', - // targetRevision: "main", - // }, - // values: { - // domain: "" - // } - // } - // ], - // adminPasswordSecretName: "argo-admin-secret" - }); - const addOns: Array = [ - new blueprints.addons.AppMeshAddOn(), - new blueprints.addons.CertManagerAddOn(), - new blueprints.addons.KubeStateMetricsAddOn(), - new blueprints.addons.PrometheusNodeExporterAddOn(), - new blueprints.addons.AdotCollectorAddOn(), - new blueprints.addons.AmpAddOn(), - new blueprints.addons.XrayAdotAddOn(), - // new blueprints.addons.CloudWatchAdotAddOn(), - new blueprints.addons.IstioBaseAddOn(), - new blueprints.addons.IstioControlPlaneAddOn(), - new blueprints.addons.CalicoOperatorAddOn(), - new blueprints.addons.MetricsServerAddOn(), - new blueprints.addons.AwsLoadBalancerControllerAddOn(), - new blueprints.addons.SecretsStoreAddOn(), - prodBootstrapArgo, - new blueprints.addons.SSMAgentAddOn(), - new blueprints.addons.NginxAddOn({ - values: { - controller: { service: { create: false } } - } - }), - new blueprints.addons.VeleroAddOn(), - new blueprints.addons.VpcCniAddOn({ - customNetworkingConfig: { - subnets: [ - blueprints.getNamedResource("secondary-cidr-subnet-0"), - blueprints.getNamedResource("secondary-cidr-subnet-1"), - blueprints.getNamedResource("secondary-cidr-subnet-2"), - ] - }, - awsVpcK8sCniCustomNetworkCfg: true, - eniConfigLabelDef: 'topology.kubernetes.io/zone' - }), - new blueprints.addons.CoreDnsAddOn(), - new blueprints.addons.KubeProxyAddOn(), - new blueprints.addons.OpaGatekeeperAddOn(), - new blueprints.addons.AckAddOn({ - id: "s3-ack", - createNamespace: true, - skipVersionValidation: true, - serviceName: blueprints.AckServiceName.S3 - }), - new blueprints.addons.KarpenterAddOn({ - requirements: [ - { key: 'node.kubernetes.io/instance-type', op: 'In', vals: ['m5.2xlarge'] }, - { key: 'topology.kubernetes.io/zone', op: 'NotIn', vals: ['us-west-2c']}, - { key: 'kubernetes.io/arch', op: 'In', vals: ['amd64','arm64']}, - { key: 'karpenter.sh/capacity-type', op: 'In', vals: ['spot']}, - ], - subnetTags: { - "Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", - }, - securityGroupTags: { - "kubernetes.io/cluster/blueprint-construct-dev": "owned", - }, - taints: [{ - key: "workload", - value: "test", - effect: "NoSchedule", - }], - consolidation: { enabled: true }, - ttlSecondsUntilExpired: 2592000, - weight: 20, - interruptionHandling: true, - limits: { - resources: { - cpu: 20, - memory: "64Gi", - } - } - }), - new blueprints.addons.AwsNodeTerminationHandlerAddOn(), - new blueprints.addons.KubeviousAddOn(), - new blueprints.addons.EbsCsiDriverAddOn({ - kmsKeys: [ - blueprints.getResource( context => new kms.Key(context.scope, "ebs-csi-driver-key", { alias: "ebs-csi-driver-key"})), - ], - } - ), - new blueprints.addons.EfsCsiDriverAddOn({replicaCount: 1}), - new blueprints.addons.KedaAddOn({ - podSecurityContextFsGroup: 1001, - securityContextRunAsGroup: 1001, - securityContextRunAsUser: 1001, - irsaRoles: ["CloudWatchFullAccess", "AmazonSQSFullAccess"] - }), - new blueprints.addons.AWSPrivateCAIssuerAddon(), - new blueprints.addons.JupyterHubAddOn({ - efsConfig: { - pvcName: "efs-persist", - removalPolicy: cdk.RemovalPolicy.DESTROY, - capacity: '10Gi', - }, - serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, - notebookStack: 'jupyter/datascience-notebook', - values: { prePuller: { hook: { enabled: false }}} - }), - new blueprints.EmrEksAddOn(), - new blueprints.AwsBatchAddOn(), - ]; + // Instantiated to for helm version check. + new blueprints.ExternalDnsAddOn({ + hostedZoneResources: [blueprints.GlobalResources.HostedZone], + }); + new blueprints.ExternalsSecretsAddOn(); - // Instantiated to for helm version check. - new blueprints.ExternalDnsAddOn({ - hostedZoneResources: [ blueprints.GlobalResources.HostedZone ] - }); - new blueprints.ExternalsSecretsAddOn(); - - const blueprintID = 'blueprint-construct-dev'; + const blueprintID = "blueprint-construct-dev"; - const userData = ec2.UserData.forLinux(); - userData.addCommands(`/etc/eks/bootstrap.sh ${blueprintID}`); + const userData = ec2.UserData.forLinux(); + userData.addCommands(`/etc/eks/bootstrap.sh ${blueprintID}`); - const clusterProvider = new blueprints.GenericClusterProvider({ - version: KubernetesVersion.V1_24, - mastersRole: blueprints.getResource(context => { - return new Role(context.scope, 'AdminRole', { assumedBy: new AccountRootPrincipal() }); - }), - managedNodeGroups: [ - { - id: "mng1", - amiType: NodegroupAmiType.AL2_X86_64, - instanceTypes: [new ec2.InstanceType('m5.4xlarge')], - diskSize: 25, - desiredSize: 2, - maxSize: 3, - nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS } - }, - { - id: "mng2-customami", - instanceTypes: [new ec2.InstanceType('t3.large')], - nodeGroupCapacityType: CapacityType.SPOT, - desiredSize: 0, - minSize: 0, - customAmi: { - machineImage: ec2.MachineImage.genericLinux({ - 'us-east-1': 'ami-08e520f5673ee0894', - 'us-west-2': 'ami-0403ff342ceb30967', - 'us-east-2': 'ami-07109d69738d6e1ee', - 'us-west-1': 'ami-07bda4b61dc470985', - 'us-gov-west-1': 'ami-0e9ebbf0d3f263e9b', - 'us-gov-east-1':'ami-033eb9bc6daf8bfb1' - }), - userData: userData, - } - } - ] + const clusterProvider = new blueprints.GenericClusterProvider({ + version: KubernetesVersion.V1_24, + mastersRole: blueprints.getResource((context) => { + return new Role(context.scope, "AdminRole", { + assumedBy: new AccountRootPrincipal(), }); - - const executionRolePolicyStatement: PolicyStatement [] = [ - new PolicyStatement({ - resources: ['*'], - actions: ['s3:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: ['glue:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: [ - 'logs:*', - ], + }), + managedNodeGroups: [ + { + id: "mng1", + amiType: NodegroupAmiType.AL2_X86_64, + instanceTypes: [new ec2.InstanceType("m5.4xlarge")], + diskSize: 25, + desiredSize: 2, + maxSize: 3, + nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, + }, + { + id: "mng2-customami", + instanceTypes: [new ec2.InstanceType("t3.large")], + nodeGroupCapacityType: CapacityType.SPOT, + desiredSize: 0, + minSize: 0, + customAmi: { + machineImage: ec2.MachineImage.genericLinux({ + "us-east-1": "ami-08e520f5673ee0894", + "us-west-2": "ami-0403ff342ceb30967", + "us-east-2": "ami-07109d69738d6e1ee", + "us-west-1": "ami-07bda4b61dc470985", + "us-gov-west-1": "ami-0e9ebbf0d3f263e9b", + "us-gov-east-1": "ami-033eb9bc6daf8bfb1", }), - ]; - - const dataTeam: blueprints.EmrEksTeamProps = { - name:'dataTeam', - virtualClusterName: 'batchJob', - virtualClusterNamespace: 'batchjob', - createNamespace: true, - executionRoles: [ - { - executionRoleIamPolicyStatement: executionRolePolicyStatement, - executionRoleName: 'myBlueprintExecRole' - } - ] - }; + userData: userData, + }, + }, + ], + }); - const batchTeam: blueprints.BatchEksTeamProps = { - name: 'batch-a', - namespace: 'aws-batch', - envName: 'batch-a-comp-env', - computeResources: { - envType: blueprints.BatchEnvType.EC2, - allocationStrategy: blueprints.BatchAllocationStrategy.BEST, - priority: 10, - minvCpus: 0, - maxvCpus: 128, - instanceTypes: ["m5", "c4.4xlarge"] - }, - jobQueueName: 'team-a-job-queue', - }; + const executionRolePolicyStatement: PolicyStatement[] = [ + new PolicyStatement({ + resources: ["*"], + actions: ["s3:*"], + }), + new PolicyStatement({ + resources: ["*"], + actions: ["glue:*"], + }), + new PolicyStatement({ + resources: ["*"], + actions: ["logs:*"], + }), + ]; - blueprints.EksBlueprint.builder() - .addOns(...addOns) - .resourceProvider(blueprints.GlobalResources.Vpc, new VpcProvider(undefined,"100.64.0.0/16", ["100.64.0.0/24","100.64.1.0/24","100.64.2.0/24"])) - .clusterProvider(clusterProvider) - .teams(...teams, new blueprints.EmrEksTeam(dataTeam), new blueprints.BatchEksTeam(batchTeam)) - .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) - .build(scope, blueprintID, props); - } -} \ No newline at end of file + const dataTeam: blueprints.EmrEksTeamProps = { + name: "dataTeam", + virtualClusterName: "batchJob", + virtualClusterNamespace: "batchjob", + createNamespace: true, + executionRoles: [ + { + executionRoleIamPolicyStatement: executionRolePolicyStatement, + executionRoleName: "myBlueprintExecRole", + }, + ], + }; + + const batchTeam: blueprints.BatchEksTeamProps = { + name: "batch-a", + namespace: "aws-batch", + envName: "batch-a-comp-env", + computeResources: { + envType: blueprints.BatchEnvType.EC2, + allocationStrategy: blueprints.BatchAllocationStrategy.BEST, + priority: 10, + minvCpus: 0, + maxvCpus: 128, + instanceTypes: ["m5", "c4.4xlarge"], + }, + jobQueueName: "team-a-job-queue", + }; + + blueprints.EksBlueprint.builder() + .addOns(...addOns) + .resourceProvider( + blueprints.GlobalResources.Vpc, + new VpcProvider(undefined, "100.64.0.0/16", [ + "100.64.0.0/24", + "100.64.1.0/24", + "100.64.2.0/24", + ]) + ) + .clusterProvider(clusterProvider) + .teams( + ...teams, + new blueprints.EmrEksTeam(dataTeam), + new blueprints.BatchEksTeam(batchTeam) + ) + .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) + .build(scope, blueprintID, props); + } +} From ef6b0fa1f188f029edc0b2d434fc83e54edb0730 Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 8 Mar 2023 17:22:48 -0600 Subject: [PATCH 14/33] fix for fluent bit --- examples/blueprint-construct/index.ts | 1 + lib/addons/aws-for-fluent-bit/index.ts | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 5b15bd019..a1220bb07 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -163,6 +163,7 @@ export default class BlueprintConstruct { }), new blueprints.EmrEksAddOn(), new blueprints.AwsBatchAddOn(), + new blueprints.AwsForFluentBitAddOn(), ]; // Instantiated to for helm version check. diff --git a/lib/addons/aws-for-fluent-bit/index.ts b/lib/addons/aws-for-fluent-bit/index.ts index ac23073f7..89ed028e8 100644 --- a/lib/addons/aws-for-fluent-bit/index.ts +++ b/lib/addons/aws-for-fluent-bit/index.ts @@ -21,7 +21,7 @@ const defaultProps: AwsForFluentBitAddOnProps = { name: 'fluent-bit', chart: 'aws-for-fluent-bit', release: "blueprints-addon-aws-for-fluent-bit", - version: '0.1.11', + version: '0.1.23', repository: 'https://aws.github.io/eks-charts', namespace: 'kube-system', values: {} @@ -45,10 +45,7 @@ export class AwsForFluentBitAddOn extends HelmAddOn { deploy(clusterInfo: ClusterInfo): Promise { const cluster = clusterInfo.cluster; - - // Create the FluentBit namespace. const namespace = this.options.namespace; - createNamespace(this.options.namespace!, cluster, true); // Create the FluentBut service account. const serviceAccountName = 'aws-for-fluent-bit-sa'; @@ -71,6 +68,7 @@ export class AwsForFluentBitAddOn extends HelmAddOn { }; const helmChart = this.addHelmChart(clusterInfo, values); + helmChart.node.addDependency(sa); return Promise.resolve(helmChart); } } From 0e287f7179ae88333cb2caf533e0f79bb53facae Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 11:54:41 -0600 Subject: [PATCH 15/33] Adding CreateEfsFileSystemProvider tests --- lib/resource-providers/efs.ts | 6 +- test/resource-providers/efs.test.ts | 103 ++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 test/resource-providers/efs.test.ts diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 7bb0a1d2b..96bf98b70 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -19,7 +19,7 @@ export interface LookupEfsFileSystemProps { /** * EFS resource provider. * - * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + * @param name The name of the EFS file system to create. * @param efsProps The props used for the file system. * @param removalPolicy The removal policy to use for the EFS file system. */ @@ -80,8 +80,8 @@ export class CreateEfsFileSystemProvider /** * Pass an EFS file system name and id to lookup an existing EFS file system. - * @param name The name of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. - * @param fileSystemId The id of the EFS file system to lookup an existing EFS file system in the deployment target. If omitted, a file system will be created. + * @param name The name of the EFS file system to lookup an existing EFS file system. + * @param fileSystemId The id of the EFS file system to lookup an existing EFS file system. */ export class LookupEfsFileSystemProvider implements ResourceProvider diff --git a/test/resource-providers/efs.test.ts b/test/resource-providers/efs.test.ts new file mode 100644 index 000000000..69ef4828b --- /dev/null +++ b/test/resource-providers/efs.test.ts @@ -0,0 +1,103 @@ +import { App } from "aws-cdk-lib"; +import { Match, Template } from "aws-cdk-lib/assertions"; +import * as blueprints from "../../lib"; +import { GlobalResources } from "../../lib"; + +describe("EfsFileSystemProvider", () => { + test("Stack is created with EFS file system with no encryption", () => { + // Given + const app = new App(); + const stack = blueprints.EksBlueprint.builder() + .resourceProvider(GlobalResources.Vpc, new blueprints.VpcProvider()) + .resourceProvider( + "my-efs-file-system-not-encrypted", + new blueprints.CreateEfsFileSystemProvider({ + name: "my-efs-file-system-not-encrypted", + efsProps: { + encrypted: false, + }, + }) + ) + .account("123456789012") + .region("us-east-1") + .build(app, "east-test-1"); + + // When + const template = Template.fromStack(stack); + + // Then + template.hasResource("AWS::EFS::FileSystem", { + Properties: { + Encrypted: false, + }, + }); + }); + test("Stack is created with EFS file system encrypted by AWS managed KMS key", () => { + // Given + const app = new App(); + const stack = blueprints.EksBlueprint.builder() + .resourceProvider(GlobalResources.Vpc, new blueprints.VpcProvider()) + .resourceProvider( + "my-efs-file-system-encrypted", + new blueprints.CreateEfsFileSystemProvider({ + name: "my-efs-file-system-encrypted", + efsProps: { + encrypted: true, + }, + }) + ) + .account("123456789012") + .region("us-east-1") + .build(app, "east-test-1"); + + // When + const template = Template.fromStack(stack); + + // Then + template.hasResource("AWS::EFS::FileSystem", { + Properties: { + Encrypted: true, + }, + }); + }); + test("Stack is created with EFS file system encrypted by specific KMS key", () => { + // Given + const efsKmsKeyName = "efs-kms-encryption-key"; + const app = new App(); + const stack = blueprints.EksBlueprint.builder() + .resourceProvider(GlobalResources.Vpc, new blueprints.VpcProvider()) + .resourceProvider( + efsKmsKeyName, + new blueprints.CreateKmsKeyProvider(efsKmsKeyName) + ) + .resourceProvider( + "my-efs-file-system-encrypted-with-kms-key", + new blueprints.CreateEfsFileSystemProvider({ + name: "my-efs-file-system-encrypted-with-kms-key", + kmsKeyResourceName: efsKmsKeyName, + efsProps: { + encrypted: true, + }, + }) + ) + .account("123456789012") + .region("us-east-1") + .build(app, "east-test-1"); + + // When + const template = Template.fromStack(stack); + + // Then + template.hasResource("AWS::EFS::FileSystem", { + Properties: { + Encrypted: true, + }, + }); + template.hasResourceProperties("AWS::EFS::FileSystem", { + KmsKeyId: {}, + }); + template.hasResourceProperties("AWS::KMS::Alias", { + AliasName: Match.stringLikeRegexp(efsKmsKeyName), + }); + }); +}); From cf094e2081df63d81838f7b295432ed7ed48605f Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 12:28:49 -0600 Subject: [PATCH 16/33] Updating the documentation --- docs/addons/efs-csi-driver.md | 10 +++++----- docs/resource-providers/index.md | 22 ++++++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/addons/efs-csi-driver.md b/docs/addons/efs-csi-driver.md index 74487d708..9420c3fd9 100644 --- a/docs/addons/efs-csi-driver.md +++ b/docs/addons/efs-csi-driver.md @@ -1,7 +1,7 @@ # EFS CSI Driver Amazon EKS Add-on The `EFS CSI Driver Amazon EKS Add-on` provides a CSI interface that allows Kubernetes clusters running on AWS to manage the lifecycle of Amazon EFS volumes for persistent storage. -EFS CSI driver supports both dynamic and static provisioning of storage +EFS CSI driver supports both dynamic and static provisioning of storage. A couple of things to note: @@ -12,7 +12,7 @@ For more information on the driver, please review the [user guide](https://docs. ## Prerequisites -- The EFS file system itself must be created in AWS separately as the driver uses the EFS for storage, but it does not create it. +- The EFS file system itself must be created in AWS separately as the driver uses the EFS for storage, but it does not create it. You can create an EFS file system using the `CreateEfsFileSystemProvider`, e.g.: `.resourceProvider("efs-file-system", new CreateEfsFileSystemProvider('efs-file-system'))` ## Usage @@ -35,7 +35,7 @@ const blueprint = blueprints.EksBlueprint.builder() - `version`: Version of the EFS CSI Driver add-on to be installed. Version 2.2.3 will be installed by default if a value is not provided - `replicaCount`: Number of replicas to be deployed. If not provided, two replicas will be deployed. Note that the number of replicas should be less than or equal to the number of nodes in the cluster otherwise some pods will be left of pending state - +- `kmsKeys`: List of KMS keys to be used for encryption-at-rest ## Validation @@ -51,8 +51,8 @@ efs-csi-node-2c29j 3/3 Running 0 155m ``` -Additionally, the [driver documentation](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) shows how to create an EFS file system to test the driver +Additionally, the [driver documentation](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) shows how to create an EFS file system to test the driver ## Functionality -Applies the EFS CSI Driver add-on to an Amazon EKS cluster. \ No newline at end of file +Applies the EFS CSI Driver add-on to an Amazon EKS cluster. diff --git a/docs/resource-providers/index.md b/docs/resource-providers/index.md index 0d65ae912..1e394ddd2 100644 --- a/docs/resource-providers/index.md +++ b/docs/resource-providers/index.md @@ -1,18 +1,18 @@ # Resource Providers -## Terminology +## Terminology **Resource** A resource is a CDK construct that implements `IResource` interface from `aws-cdk-lib` which is a generic interface for any AWS resource. An example of a resource could be a hosted zone in Route53 [`IHostedZone`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_route53.HostedZone.html), an ACM certificate [`ICertificate`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_certificatemanager.ICertificate.html), a VPC or even a DynamoDB table which could be leveraged either in add-ons or teams. **ResourceProvider** -A resource provider is a core Blueprints concept that enables customers to supply resources for add-ons, teams and/or post-deployment steps. Resources may be imported (e.g., if created outside of the platform) or created with the blueprint. +A resource provider is a core Blueprints concept that enables customers to supply resources for add-ons, teams and/or post-deployment steps. Resources may be imported (e.g., if created outside of the platform) or created with the blueprint. ## Use Cases -`ClusterAddOn` and `Team` implementations require AWS resources that can be shared across several constructs. For example, `ExternalDnsAddOn` requires an array of hosted zones that will be used for integration with Route53. `NginxAddOn` requires a certificate and hosted zone (for DNS validation) in order to use TLS termination. VPC may be used inside add-ons and team constructs to look up VPC CIDR and subnets. +`ClusterAddOn` and `Team` implementations require AWS resources that can be shared across several constructs. For example, `ExternalDnsAddOn` requires an array of hosted zones that will be used for integration with Route53. `NginxAddOn` requires a certificate and hosted zone (for DNS validation) in order to use TLS termination. VPC may be used inside add-ons and team constructs to look up VPC CIDR and subnets. -The Blueprints framework provides ability to register a resource provider under an arbitrary name and make it available in the resource context, which is available to all add-ons and teams. With this capability, customers can either use existing resource providers or create their own and reference the provided resources inside add-ons, teams or other resource providers. +The Blueprints framework provides ability to register a resource provider under an arbitrary name and make it available in the resource context, which is available to all add-ons and teams. With this capability, customers can either use existing resource providers or create their own and reference the provided resources inside add-ons, teams or other resource providers. Resource providers may depend on resources provided by other resource providers. For example, `CertificateResourceProvider` relies on a hosted zone resource, which is expected to be supplied by another provider. @@ -20,9 +20,9 @@ Example use cases: 1. As a platform user, I must create a VPC using my enterprise standards and leverage it for the EKS Blueprint. Solution: create an implementation of `ResourceProvider` (or leverage an existing one) and register it with the blueprint (see Usage). -2. As a platform user, I need to use an existing hosted zone for all external DNS names used with ingress objects of my workloads. Solution: use a predefined `ImportHostedZoneProvider` or `LookupHostedZoneProvider` to reference the existing hosted zone. +2. As a platform user, I need to use an existing hosted zone for all external DNS names used with ingress objects of my workloads. Solution: use a predefined `ImportHostedZoneProvider` or `LookupHostedZoneProvider` to reference the existing hosted zone. -3. As a platform user, I need to create an S3 bucket and use it in one or more `Team` implementations. Solution: create an implementation for an S3 Bucket resource provider and use the supplied resource inside teams. +3. As a platform user, I need to create an S3 bucket and use it in one or more `Team` implementations. Solution: create an implementation for an S3 Bucket resource provider and use the supplied resource inside teams. ## Contracts @@ -119,7 +119,6 @@ export class ClusterInfo { } ``` - ## Usage **Registering Resource Providers for a Blueprint** @@ -140,6 +139,8 @@ blueprints.EksBlueprint.builder() // Register certificate GlobalResources.Certificate name and reference the hosted zone registered in the previous step .resourceProvider(GlobalResources.Certificate, new CreateCertificateProvider('domain-wildcard-cert', '*.my.domain.com', GlobalResources.HostedZone)) .resourceProvider("private-ca", new CreateCertificateProvider('internal-wildcard-cert', '*.myinternal.domain.com', "internal-hosted-zone")) + // Create EFS file system and register it under the name of efs-file-system + .resourceProvider("efs-file-system", new CreateEfsFileSystemProvider('efs-file-system')) .addOns(new AwsLoadBalancerControllerAddOn()) // Use hosted zone for External DNS .addOns(new ExternalDnsAddOn({hostedZoneResources: [GlobalResources.HostedZone]})) @@ -173,9 +174,10 @@ blueprints.EksBlueprint.builder() .teams(...) .build(app, 'stack-with-resource-providers'); ``` + ## Using Resource Providers with CDK Constructs -Some constructs used in the `EKSBlueprint` stack are standard CDK constructs that accept CDK resources. +Some constructs used in the `EKSBlueprint` stack are standard CDK constructs that accept CDK resources. For example, `GenericClusterProvider` (which is the basis for all cluster providers) allows passing resources like `IRole`, `SecurityGroup` and other properties that customers may find inconvenient to define with a builder pattern. @@ -201,7 +203,7 @@ blueprints.EksBlueprint.builder() } ``` -Example with a named resource: +Example with a named resource: ```typescript const clusterProvider = new blueprints.GenericClusterProvider({ @@ -244,6 +246,7 @@ blueprints.EksBlueprint.builder() ``` 4. Use the resource inside a custom add-on: + ```typescript class MyCustomAddOn implements blueprints.ClusterAddOn { deploy(clusterInfo: ClusterInfo): void | Promise { @@ -253,4 +256,3 @@ class MyCustomAddOn implements blueprints.ClusterAddOn { } ``` - From 83abad58654170dcfdcd48a2eda4c053801ac0de Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 12:55:49 -0600 Subject: [PATCH 17/33] Adding CfnOutput and LookupEfsFileSystemProvider with unit tests --- lib/resource-providers/efs.ts | 7 +++++++ test/resource-providers/efs.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 96bf98b70..946ebedcb 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -1,3 +1,4 @@ +import { CfnOutput } from "aws-cdk-lib"; import * as ec2 from "aws-cdk-lib/aws-ec2"; import * as efs from "aws-cdk-lib/aws-efs"; import * as kms from "aws-cdk-lib/aws-kms"; @@ -74,6 +75,9 @@ export class CreateEfsFileSystemProvider ...this.options.efsProps, } ); + new CfnOutput(context.scope, "EfsFileSystemId", { + value: efsFileSystem.fileSystemId, + }); return efsFileSystem; } } @@ -116,6 +120,9 @@ export class LookupEfsFileSystemProvider if (!efsFileSystem) { throw new Error("EFS file system not found"); } + new CfnOutput(context.scope, "EfsFileSystemId", { + value: efsFileSystem.fileSystemId, + }); return efsFileSystem; } } diff --git a/test/resource-providers/efs.test.ts b/test/resource-providers/efs.test.ts index 69ef4828b..2ba79d60f 100644 --- a/test/resource-providers/efs.test.ts +++ b/test/resource-providers/efs.test.ts @@ -100,4 +100,26 @@ describe("EfsFileSystemProvider", () => { AliasName: Match.stringLikeRegexp(efsKmsKeyName), }); }); + test("Stack created with lookup EFS file system resource provider", () => { + // Given + const efsFileSystemName = "efs-file-system"; + const app = new App(); + const stack = blueprints.EksBlueprint.builder() + .resourceProvider( + efsFileSystemName, + new blueprints.LookupEfsFileSystemProvider({ + name: efsFileSystemName, + fileSystemId: "fs-12345678", + }) + ) + .account("123456789012") + .region("us-east-1") + .build(app, "east-test-1"); + + // When + const template = Template.fromStack(stack); + + // Then + template.hasOutput("EfsFileSystemId", { Value: "fs-12345678" }); + }); }); From ac28acdaad8a0a2067b471339067b36bb968f6c6 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:14:30 -0600 Subject: [PATCH 18/33] autoformat: changed lines only --- examples/blueprint-construct/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index d8a272be9..40972934a 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -5,17 +5,17 @@ import { KubernetesVersion, NodegroupAmiType, } from "aws-cdk-lib/aws-eks"; +import * as kms from "aws-cdk-lib/aws-kms"; import { AccountRootPrincipal, PolicyStatement, Role, } from "aws-cdk-lib/aws-iam"; -import * as kms from "aws-cdk-lib/aws-kms"; import { Construct } from "constructs"; import * as blueprints from "../../lib"; -import { VpcProvider } from "../../lib"; import { logger, userLog } from "../../lib/utils"; import * as team from "../teams"; +import { VpcProvider } from "../../lib"; const burnhamManifestDir = "./examples/teams/team-burnham/"; const rikerManifestDir = "./examples/teams/team-riker/"; @@ -165,15 +165,15 @@ export default class BlueprintConstruct { ], }), new blueprints.addons.EfsCsiDriverAddOn({ - replicaCount: 1, kmsKeys: [ blueprints.getResource( (context) => - new kms.Key(context.scope, "efs-csi-driver-key", { - alias: "efs-csi-driver-key", + new kms.Key(context.scope, "ebs-csi-driver-key", { + alias: "ebs-csi-driver-key", }) ), ], + replicaCount: 1, }), new blueprints.addons.KedaAddOn({ podSecurityContextFsGroup: 1001, From b2e3255c1b329c51ce88f7112e0e9206696a22d7 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:16:45 -0600 Subject: [PATCH 19/33] autoformat: changed lines only --- examples/blueprint-construct/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 40972934a..cef799dde 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -165,6 +165,7 @@ export default class BlueprintConstruct { ], }), new blueprints.addons.EfsCsiDriverAddOn({ + replicaCount: 1, kmsKeys: [ blueprints.getResource( (context) => @@ -173,7 +174,6 @@ export default class BlueprintConstruct { }) ), ], - replicaCount: 1, }), new blueprints.addons.KedaAddOn({ podSecurityContextFsGroup: 1001, From 11744e302759147cd80fabba90e486d918e7d6ca Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:18:51 -0600 Subject: [PATCH 20/33] Reversing the chages to blueprint-construct/index.ts --- examples/blueprint-construct/index.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index cef799dde..4f2784fd6 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -164,17 +164,7 @@ export default class BlueprintConstruct { ), ], }), - new blueprints.addons.EfsCsiDriverAddOn({ - replicaCount: 1, - kmsKeys: [ - blueprints.getResource( - (context) => - new kms.Key(context.scope, "ebs-csi-driver-key", { - alias: "ebs-csi-driver-key", - }) - ), - ], - }), + new blueprints.addons.EfsCsiDriverAddOn({ replicaCount: 1 }), new blueprints.addons.KedaAddOn({ podSecurityContextFsGroup: 1001, securityContextRunAsGroup: 1001, From 16fd7d6f49ca9e46f2b3b4650ecaf751a7c91243 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:21:06 -0600 Subject: [PATCH 21/33] autoformat: changed lines only --- examples/blueprint-construct/index.ts | 537 ++++++++++++-------------- 1 file changed, 255 insertions(+), 282 deletions(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 4f2784fd6..41943e844 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -1,300 +1,273 @@ -import * as cdk from "aws-cdk-lib"; +import * as cdk from 'aws-cdk-lib'; import * as ec2 from "aws-cdk-lib/aws-ec2"; -import { - CapacityType, - KubernetesVersion, - NodegroupAmiType, -} from "aws-cdk-lib/aws-eks"; -import * as kms from "aws-cdk-lib/aws-kms"; -import { - AccountRootPrincipal, - PolicyStatement, - Role, -} from "aws-cdk-lib/aws-iam"; +import { CapacityType, KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import { AccountRootPrincipal, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; import { Construct } from "constructs"; -import * as blueprints from "../../lib"; -import { logger, userLog } from "../../lib/utils"; -import * as team from "../teams"; -import { VpcProvider } from "../../lib"; +import * as blueprints from '../../lib'; +import { logger, userLog } from '../../lib/utils'; +import * as team from '../teams'; +import { VpcProvider } from '../../lib'; -const burnhamManifestDir = "./examples/teams/team-burnham/"; -const rikerManifestDir = "./examples/teams/team-riker/"; +const burnhamManifestDir = './examples/teams/team-burnham/'; +const rikerManifestDir = './examples/teams/team-riker/'; const teamManifestDirList = [burnhamManifestDir, rikerManifestDir]; export interface BlueprintConstructProps { - /** - * Id - */ - id: string; + /** + * Id + */ + id: string } export default class BlueprintConstruct { - constructor(scope: Construct, props: cdk.StackProps) { - blueprints.HelmAddOn.validateHelmVersions = true; - blueprints.HelmAddOn.failOnVersionValidation = false; - logger.settings.minLevel = 3; - userLog.settings.minLevel = 2; + constructor(scope: Construct, props: cdk.StackProps) { - // TODO: fix IAM user provisioning for admin user - // Setup platform team. - //const account = props.env!.account! - // const platformTeam = new team.TeamPlatform(account) - // Teams for the cluster. - const teams: Array = [ - new team.TeamTroi(), - new team.TeamRiker(scope, teamManifestDirList[1]), - new team.TeamBurnham(scope, teamManifestDirList[0]), - new team.TeamPlatform(process.env.CDK_DEFAULT_ACCOUNT!), - ]; - const prodBootstrapArgo = new blueprints.addons.ArgoCDAddOn({ - // TODO: enabling this cause stack deletion failure, known issue: - // https://github.com/aws-quickstart/cdk-eks-blueprints/blob/main/docs/addons/argo-cd.md#known-issues - // bootstrapRepo: { - // repoUrl: 'https://github.com/aws-samples/eks-blueprints-add-ons.git', - // path: 'chart', - // targetRevision: "eks-blueprints-cdk", - // }, - // workloadApplications: [ - // { - // name: "micro-services", - // namespace: "argocd", - // repository: { - // repoUrl: 'https://github.com/aws-samples/eks-blueprints-workloads.git', - // path: 'envs/dev', - // targetRevision: "main", - // }, - // values: { - // domain: "" - // } - // } - // ], - // adminPasswordSecretName: "argo-admin-secret" - }); - const addOns: Array = [ - new blueprints.addons.AppMeshAddOn(), - new blueprints.addons.CertManagerAddOn(), - new blueprints.addons.KubeStateMetricsAddOn(), - new blueprints.addons.PrometheusNodeExporterAddOn(), - new blueprints.addons.AdotCollectorAddOn(), - new blueprints.addons.AmpAddOn(), - new blueprints.addons.XrayAdotAddOn(), - // new blueprints.addons.CloudWatchAdotAddOn(), - new blueprints.addons.IstioBaseAddOn(), - new blueprints.addons.IstioControlPlaneAddOn(), - new blueprints.addons.CalicoOperatorAddOn(), - new blueprints.addons.MetricsServerAddOn(), - new blueprints.addons.AwsLoadBalancerControllerAddOn(), - new blueprints.addons.SecretsStoreAddOn(), - prodBootstrapArgo, - new blueprints.addons.SSMAgentAddOn(), - new blueprints.addons.NginxAddOn({ - values: { - controller: { service: { create: false } }, - }, - }), - new blueprints.addons.VeleroAddOn(), - new blueprints.addons.VpcCniAddOn({ - customNetworkingConfig: { - subnets: [ - blueprints.getNamedResource("secondary-cidr-subnet-0"), - blueprints.getNamedResource("secondary-cidr-subnet-1"), - blueprints.getNamedResource("secondary-cidr-subnet-2"), - ], - }, - awsVpcK8sCniCustomNetworkCfg: true, - eniConfigLabelDef: "topology.kubernetes.io/zone", - }), - new blueprints.addons.CoreDnsAddOn(), - new blueprints.addons.KubeProxyAddOn(), - new blueprints.addons.OpaGatekeeperAddOn(), - new blueprints.addons.AckAddOn({ - id: "s3-ack", - createNamespace: true, - skipVersionValidation: true, - serviceName: blueprints.AckServiceName.S3, - }), - new blueprints.addons.KarpenterAddOn({ - requirements: [ - { - key: "node.kubernetes.io/instance-type", - op: "In", - vals: ["m5.2xlarge"], - }, - { - key: "topology.kubernetes.io/zone", - op: "NotIn", - vals: ["us-west-2c"], - }, - { key: "kubernetes.io/arch", op: "In", vals: ["amd64", "arm64"] }, - { key: "karpenter.sh/capacity-type", op: "In", vals: ["spot"] }, - ], - subnetTags: { - Name: "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", - }, - securityGroupTags: { - "kubernetes.io/cluster/blueprint-construct-dev": "owned", - }, - taints: [ - { - key: "workload", - value: "test", - effect: "NoSchedule", - }, - ], - consolidation: { enabled: true }, - ttlSecondsUntilExpired: 2592000, - weight: 20, - interruptionHandling: true, - limits: { - resources: { - cpu: 20, - memory: "64Gi", - }, - }, - }), - new blueprints.addons.AwsNodeTerminationHandlerAddOn(), - new blueprints.addons.KubeviousAddOn(), - new blueprints.addons.EbsCsiDriverAddOn({ - kmsKeys: [ - blueprints.getResource( - (context) => - new kms.Key(context.scope, "ebs-csi-driver-key", { - alias: "ebs-csi-driver-key", - }) - ), - ], - }), - new blueprints.addons.EfsCsiDriverAddOn({ replicaCount: 1 }), - new blueprints.addons.KedaAddOn({ - podSecurityContextFsGroup: 1001, - securityContextRunAsGroup: 1001, - securityContextRunAsUser: 1001, - irsaRoles: ["CloudWatchFullAccess", "AmazonSQSFullAccess"], - }), - new blueprints.addons.AWSPrivateCAIssuerAddon(), - new blueprints.addons.JupyterHubAddOn({ - efsConfig: { - pvcName: "efs-persist", - removalPolicy: cdk.RemovalPolicy.DESTROY, - capacity: "10Gi", - }, - serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, - notebookStack: "jupyter/datascience-notebook", - values: { prePuller: { hook: { enabled: false } } }, - }), - new blueprints.EmrEksAddOn(), - new blueprints.AwsBatchAddOn(), - ]; + blueprints.HelmAddOn.validateHelmVersions = true; + blueprints.HelmAddOn.failOnVersionValidation = false; + logger.settings.minLevel = 3; + userLog.settings.minLevel = 2; - // Instantiated to for helm version check. - new blueprints.ExternalDnsAddOn({ - hostedZoneResources: [blueprints.GlobalResources.HostedZone], - }); - new blueprints.ExternalsSecretsAddOn(); + // TODO: fix IAM user provisioning for admin user + // Setup platform team. + //const account = props.env!.account! + // const platformTeam = new team.TeamPlatform(account) + // Teams for the cluster. + const teams: Array = [ + new team.TeamTroi, + new team.TeamRiker(scope, teamManifestDirList[1]), + new team.TeamBurnham(scope, teamManifestDirList[0]), + new team.TeamPlatform(process.env.CDK_DEFAULT_ACCOUNT!) + ]; + const prodBootstrapArgo = new blueprints.addons.ArgoCDAddOn({ + // TODO: enabling this cause stack deletion failure, known issue: + // https://github.com/aws-quickstart/cdk-eks-blueprints/blob/main/docs/addons/argo-cd.md#known-issues + // bootstrapRepo: { + // repoUrl: 'https://github.com/aws-samples/eks-blueprints-add-ons.git', + // path: 'chart', + // targetRevision: "eks-blueprints-cdk", + // }, + // workloadApplications: [ + // { + // name: "micro-services", + // namespace: "argocd", + // repository: { + // repoUrl: 'https://github.com/aws-samples/eks-blueprints-workloads.git', + // path: 'envs/dev', + // targetRevision: "main", + // }, + // values: { + // domain: "" + // } + // } + // ], + // adminPasswordSecretName: "argo-admin-secret" + }); + const addOns: Array = [ + new blueprints.addons.AppMeshAddOn(), + new blueprints.addons.CertManagerAddOn(), + new blueprints.addons.KubeStateMetricsAddOn(), + new blueprints.addons.PrometheusNodeExporterAddOn(), + new blueprints.addons.AdotCollectorAddOn(), + new blueprints.addons.AmpAddOn(), + new blueprints.addons.XrayAdotAddOn(), + // new blueprints.addons.CloudWatchAdotAddOn(), + new blueprints.addons.IstioBaseAddOn(), + new blueprints.addons.IstioControlPlaneAddOn(), + new blueprints.addons.CalicoOperatorAddOn(), + new blueprints.addons.MetricsServerAddOn(), + new blueprints.addons.AwsLoadBalancerControllerAddOn(), + new blueprints.addons.SecretsStoreAddOn(), + prodBootstrapArgo, + new blueprints.addons.SSMAgentAddOn(), + new blueprints.addons.NginxAddOn({ + values: { + controller: { service: { create: false } } + } + }), + new blueprints.addons.VeleroAddOn(), + new blueprints.addons.VpcCniAddOn({ + customNetworkingConfig: { + subnets: [ + blueprints.getNamedResource("secondary-cidr-subnet-0"), + blueprints.getNamedResource("secondary-cidr-subnet-1"), + blueprints.getNamedResource("secondary-cidr-subnet-2"), + ] + }, + awsVpcK8sCniCustomNetworkCfg: true, + eniConfigLabelDef: 'topology.kubernetes.io/zone' + }), + new blueprints.addons.CoreDnsAddOn(), + new blueprints.addons.KubeProxyAddOn(), + new blueprints.addons.OpaGatekeeperAddOn(), + new blueprints.addons.AckAddOn({ + id: "s3-ack", + createNamespace: true, + skipVersionValidation: true, + serviceName: blueprints.AckServiceName.S3 + }), + new blueprints.addons.KarpenterAddOn({ + requirements: [ + { key: 'node.kubernetes.io/instance-type', op: 'In', vals: ['m5.2xlarge'] }, + { key: 'topology.kubernetes.io/zone', op: 'NotIn', vals: ['us-west-2c']}, + { key: 'kubernetes.io/arch', op: 'In', vals: ['amd64','arm64']}, + { key: 'karpenter.sh/capacity-type', op: 'In', vals: ['spot']}, + ], + subnetTags: { + "Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", + }, + securityGroupTags: { + "kubernetes.io/cluster/blueprint-construct-dev": "owned", + }, + taints: [{ + key: "workload", + value: "test", + effect: "NoSchedule", + }], + consolidation: { enabled: true }, + ttlSecondsUntilExpired: 2592000, + weight: 20, + interruptionHandling: true, + limits: { + resources: { + cpu: 20, + memory: "64Gi", + } + } + }), + new blueprints.addons.AwsNodeTerminationHandlerAddOn(), + new blueprints.addons.KubeviousAddOn(), + new blueprints.addons.EbsCsiDriverAddOn({ + kmsKeys: [ + blueprints.getResource( context => new kms.Key(context.scope, "ebs-csi-driver-key", { alias: "ebs-csi-driver-key"})), + ], + } + ), + new blueprints.addons.EfsCsiDriverAddOn({ + replicaCount: 1, + kmsKeys: [ + blueprints.getResource( context => new kms.Key(context.scope, "ebs-csi-driver-key", { alias: "ebs-csi-driver-key"})), + ], + }), + new blueprints.addons.KedaAddOn({ + podSecurityContextFsGroup: 1001, + securityContextRunAsGroup: 1001, + securityContextRunAsUser: 1001, + irsaRoles: ["CloudWatchFullAccess", "AmazonSQSFullAccess"] + }), + new blueprints.addons.AWSPrivateCAIssuerAddon(), + new blueprints.addons.JupyterHubAddOn({ + efsConfig: { + pvcName: "efs-persist", + removalPolicy: cdk.RemovalPolicy.DESTROY, + capacity: '10Gi', + }, + serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, + notebookStack: 'jupyter/datascience-notebook', + values: { prePuller: { hook: { enabled: false }}} + }), + new blueprints.EmrEksAddOn(), + new blueprints.AwsBatchAddOn(), + ]; - const blueprintID = "blueprint-construct-dev"; + // Instantiated to for helm version check. + new blueprints.ExternalDnsAddOn({ + hostedZoneResources: [ blueprints.GlobalResources.HostedZone ] + }); + new blueprints.ExternalsSecretsAddOn(); + + const blueprintID = 'blueprint-construct-dev'; - const userData = ec2.UserData.forLinux(); - userData.addCommands(`/etc/eks/bootstrap.sh ${blueprintID}`); + const userData = ec2.UserData.forLinux(); + userData.addCommands(`/etc/eks/bootstrap.sh ${blueprintID}`); - const clusterProvider = new blueprints.GenericClusterProvider({ - version: KubernetesVersion.V1_24, - mastersRole: blueprints.getResource((context) => { - return new Role(context.scope, "AdminRole", { - assumedBy: new AccountRootPrincipal(), - }); - }), - managedNodeGroups: [ - { - id: "mng1", - amiType: NodegroupAmiType.AL2_X86_64, - instanceTypes: [new ec2.InstanceType("m5.4xlarge")], - diskSize: 25, - desiredSize: 2, - maxSize: 3, - nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, - }, - { - id: "mng2-customami", - instanceTypes: [new ec2.InstanceType("t3.large")], - nodeGroupCapacityType: CapacityType.SPOT, - desiredSize: 0, - minSize: 0, - customAmi: { - machineImage: ec2.MachineImage.genericLinux({ - "us-east-1": "ami-08e520f5673ee0894", - "us-west-2": "ami-0403ff342ceb30967", - "us-east-2": "ami-07109d69738d6e1ee", - "us-west-1": "ami-07bda4b61dc470985", - "us-gov-west-1": "ami-0e9ebbf0d3f263e9b", - "us-gov-east-1": "ami-033eb9bc6daf8bfb1", + const clusterProvider = new blueprints.GenericClusterProvider({ + version: KubernetesVersion.V1_24, + mastersRole: blueprints.getResource(context => { + return new Role(context.scope, 'AdminRole', { assumedBy: new AccountRootPrincipal() }); }), - userData: userData, - }, - }, - ], - }); - - const executionRolePolicyStatement: PolicyStatement[] = [ - new PolicyStatement({ - resources: ["*"], - actions: ["s3:*"], - }), - new PolicyStatement({ - resources: ["*"], - actions: ["glue:*"], - }), - new PolicyStatement({ - resources: ["*"], - actions: ["logs:*"], - }), - ]; + managedNodeGroups: [ + { + id: "mng1", + amiType: NodegroupAmiType.AL2_X86_64, + instanceTypes: [new ec2.InstanceType('m5.4xlarge')], + diskSize: 25, + desiredSize: 2, + maxSize: 3, + nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS } + }, + { + id: "mng2-customami", + instanceTypes: [new ec2.InstanceType('t3.large')], + nodeGroupCapacityType: CapacityType.SPOT, + desiredSize: 0, + minSize: 0, + customAmi: { + machineImage: ec2.MachineImage.genericLinux({ + 'us-east-1': 'ami-08e520f5673ee0894', + 'us-west-2': 'ami-0403ff342ceb30967', + 'us-east-2': 'ami-07109d69738d6e1ee', + 'us-west-1': 'ami-07bda4b61dc470985', + 'us-gov-west-1': 'ami-0e9ebbf0d3f263e9b', + 'us-gov-east-1':'ami-033eb9bc6daf8bfb1' + }), + userData: userData, + } + } + ] + }); - const dataTeam: blueprints.EmrEksTeamProps = { - name: "dataTeam", - virtualClusterName: "batchJob", - virtualClusterNamespace: "batchjob", - createNamespace: true, - executionRoles: [ - { - executionRoleIamPolicyStatement: executionRolePolicyStatement, - executionRoleName: "myBlueprintExecRole", - }, - ], - }; + const executionRolePolicyStatement: PolicyStatement [] = [ + new PolicyStatement({ + resources: ['*'], + actions: ['s3:*'], + }), + new PolicyStatement({ + resources: ['*'], + actions: ['glue:*'], + }), + new PolicyStatement({ + resources: ['*'], + actions: [ + 'logs:*', + ], + }), + ]; + + const dataTeam: blueprints.EmrEksTeamProps = { + name:'dataTeam', + virtualClusterName: 'batchJob', + virtualClusterNamespace: 'batchjob', + createNamespace: true, + executionRoles: [ + { + executionRoleIamPolicyStatement: executionRolePolicyStatement, + executionRoleName: 'myBlueprintExecRole' + } + ] + }; - const batchTeam: blueprints.BatchEksTeamProps = { - name: "batch-a", - namespace: "aws-batch", - envName: "batch-a-comp-env", - computeResources: { - envType: blueprints.BatchEnvType.EC2, - allocationStrategy: blueprints.BatchAllocationStrategy.BEST, - priority: 10, - minvCpus: 0, - maxvCpus: 128, - instanceTypes: ["m5", "c4.4xlarge"], - }, - jobQueueName: "team-a-job-queue", - }; + const batchTeam: blueprints.BatchEksTeamProps = { + name: 'batch-a', + namespace: 'aws-batch', + envName: 'batch-a-comp-env', + computeResources: { + envType: blueprints.BatchEnvType.EC2, + allocationStrategy: blueprints.BatchAllocationStrategy.BEST, + priority: 10, + minvCpus: 0, + maxvCpus: 128, + instanceTypes: ["m5", "c4.4xlarge"] + }, + jobQueueName: 'team-a-job-queue', + }; - blueprints.EksBlueprint.builder() - .addOns(...addOns) - .resourceProvider( - blueprints.GlobalResources.Vpc, - new VpcProvider(undefined, "100.64.0.0/16", [ - "100.64.0.0/24", - "100.64.1.0/24", - "100.64.2.0/24", - ]) - ) - .clusterProvider(clusterProvider) - .teams( - ...teams, - new blueprints.EmrEksTeam(dataTeam), - new blueprints.BatchEksTeam(batchTeam) - ) - .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) - .build(scope, blueprintID, props); - } -} + blueprints.EksBlueprint.builder() + .addOns(...addOns) + .resourceProvider(blueprints.GlobalResources.Vpc, new VpcProvider(undefined,"100.64.0.0/16", ["100.64.0.0/24","100.64.1.0/24","100.64.2.0/24"])) + .clusterProvider(clusterProvider) + .teams(...teams, new blueprints.EmrEksTeam(dataTeam), new blueprints.BatchEksTeam(batchTeam)) + .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) + .build(scope, blueprintID, props); + } +} \ No newline at end of file From e72001555e4e440689ef582877b0e708151f1bf9 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:33:42 -0600 Subject: [PATCH 22/33] autoformat: changed lines only --- lib/addons/efs-csi-driver/index.ts | 10 +++++----- lib/resource-providers/index.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/addons/efs-csi-driver/index.ts b/lib/addons/efs-csi-driver/index.ts index 1b05b51a3..5978c69f3 100644 --- a/lib/addons/efs-csi-driver/index.ts +++ b/lib/addons/efs-csi-driver/index.ts @@ -1,11 +1,11 @@ -import * as iam from "aws-cdk-lib/aws-iam"; -import * as kms from "aws-cdk-lib/aws-kms"; import { Construct } from "constructs"; -import { ClusterInfo, Values } from "../../spi"; -import { setPath } from "../../utils"; -import { registries } from "../../utils/registry-utils"; +import {ClusterInfo, Values} from "../../spi"; import { HelmAddOn, HelmAddOnUserProps } from "../helm-addon"; import { getEfsDriverPolicyStatements } from "./iam-policy"; +import { registries } from "../../utils/registry-utils"; +import * as iam from "aws-cdk-lib/aws-iam"; +import {setPath} from "../../utils"; +import * as kms from "aws-cdk-lib/aws-kms"; const EFS_CSI_DRIVER = "aws-efs-csi-driver"; diff --git a/lib/resource-providers/index.ts b/lib/resource-providers/index.ts index 2f9dc3485..14e779c09 100644 --- a/lib/resource-providers/index.ts +++ b/lib/resource-providers/index.ts @@ -1,7 +1,7 @@ export * from './certificate'; -export * from './efs'; export * from './hosted-zone'; -export * from './iam'; export * from './kms-key'; +export * from './iam'; export * from './utils'; export * from './vpc'; +export * from './efs'; \ No newline at end of file From 2f61ea09dfdd6a940264ca0d73405710fdb53e9c Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:44:03 -0600 Subject: [PATCH 23/33] autoformat: changed lines only --- lib/resource-providers/kms-key.ts | 4 +- lib/stacks/eks-blueprint-stack.ts | 711 +++++++++++------------- test/resource-providers/kms-key.test.ts | 27 +- 3 files changed, 331 insertions(+), 411 deletions(-) diff --git a/lib/resource-providers/kms-key.ts b/lib/resource-providers/kms-key.ts index 42df03116..ec4677b10 100644 --- a/lib/resource-providers/kms-key.ts +++ b/lib/resource-providers/kms-key.ts @@ -20,7 +20,7 @@ export class CreateKmsKeyProvider implements ResourceProvider { /** * Configuration options for the KMS Key. * - * @param aliasName The alias name to lookup an existing KMS Key in the deployment target, if omitted a key will be created. + * @param aliasName The alias name for the KMS Key * @param kmsKeyProps The key props used */ public constructor(aliasName?: string, kmsKeyProps?: kms.KeyProps) { @@ -46,7 +46,7 @@ export class CreateKmsKeyProvider implements ResourceProvider { /** * Pass an aliasName to lookup an existing KMS Key. * - * @param aliasName The alias name to lookup an existing KMS Key in the deployment target, if omitted a key will be created. + * @param aliasName The alias name to lookup an existing KMS Key */ export class LookupKmsKeyProvider implements ResourceProvider { private readonly aliasName: string; diff --git a/lib/stacks/eks-blueprint-stack.ts b/lib/stacks/eks-blueprint-stack.ts index 3cbfb1d3c..c50c117f6 100644 --- a/lib/stacks/eks-blueprint-stack.ts +++ b/lib/stacks/eks-blueprint-stack.ts @@ -1,100 +1,99 @@ -import * as cdk from "aws-cdk-lib"; -import { IVpc } from "aws-cdk-lib/aws-ec2"; -import { KubernetesVersion } from "aws-cdk-lib/aws-eks"; -import { Construct } from "constructs"; -import { MngClusterProvider } from "../cluster-providers/mng-cluster-provider"; -import { VpcProvider } from "../resource-providers/vpc"; -import * as spi from "../spi"; -import * as constraints from "../utils/constraints-utils"; -import * as utils from "../utils"; -import { cloneDeep } from "../utils"; +import * as cdk from 'aws-cdk-lib'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { KubernetesVersion } from 'aws-cdk-lib/aws-eks'; +import { Construct } from 'constructs'; +import { MngClusterProvider } from '../cluster-providers/mng-cluster-provider'; +import { VpcProvider } from '../resource-providers/vpc'; +import * as spi from '../spi'; +import * as constraints from '../utils/constraints-utils'; +import * as utils from '../utils'; +import { cloneDeep } from '../utils'; import { IKey } from "aws-cdk-lib/aws-kms"; -import { CreateKmsKeyProvider } from "../resource-providers/kms-key"; +import {CreateKmsKeyProvider} from "../resource-providers/kms-key"; import { ArgoGitOpsFactory } from "../addons/argocd/argo-gitops-factory"; export class EksBlueprintProps { - /** - * The id for the blueprint. - */ - readonly id: string; - - /** - * Defaults to id if not provided - */ - readonly name?: string; - - /** - * Add-ons if any. - */ - readonly addOns?: Array = []; - - /** - * Teams if any - */ - readonly teams?: Array = []; - - /** - * EC2 or Fargate are supported in the blueprint but any implementation conforming the interface - * will work - */ - readonly clusterProvider?: spi.ClusterProvider = new MngClusterProvider(); - - /** - * Kubernetes version (must be initialized for addons to work properly) - */ - readonly version?: KubernetesVersion = KubernetesVersion.V1_24; - - /** - * Named resource providers to leverage for cluster resources. - * The resource can represent Vpc, Hosting Zones or other resources, see {@link spi.ResourceType}. - * VPC for the cluster can be registered under the name of 'vpc' or as a single provider of type - */ - resourceProviders?: Map = new Map(); - - /** - * Control Plane log types to be enabled (if not passed, none) - * If wrong types are included, will throw an error. - */ - readonly enableControlPlaneLogTypes?: ControlPlaneLogType[]; - - /** - * If set to true and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), - * a default KMS encryption key will be used for envelope encryption of Kubernetes secrets (AWS managed new KMS key). - * If set to false, and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), then no secrets - * encyrption is applied. - * - * Default is true. - */ - readonly useDefaultSecretEncryption?: boolean = true; - - /** - * GitOps modes to be enabled. If not specified, GitOps mode is not enabled. - */ - readonly enableGitOpsMode?: spi.GitOpsMode; + /** + * The id for the blueprint. + */ + readonly id: string; + + /** + * Defaults to id if not provided + */ + readonly name?: string; + + /** + * Add-ons if any. + */ + readonly addOns?: Array = []; + + /** + * Teams if any + */ + readonly teams?: Array = []; + + /** + * EC2 or Fargate are supported in the blueprint but any implementation conforming the interface + * will work + */ + readonly clusterProvider?: spi.ClusterProvider = new MngClusterProvider(); + + /** + * Kubernetes version (must be initialized for addons to work properly) + */ + readonly version?: KubernetesVersion = KubernetesVersion.V1_24; + + /** + * Named resource providers to leverage for cluster resources. + * The resource can represent Vpc, Hosting Zones or other resources, see {@link spi.ResourceType}. + * VPC for the cluster can be registered under the name of 'vpc' or as a single provider of type + */ + resourceProviders?: Map = new Map(); + + /** + * Control Plane log types to be enabled (if not passed, none) + * If wrong types are included, will throw an error. + */ + readonly enableControlPlaneLogTypes?: ControlPlaneLogType[]; + + /** + * If set to true and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), + * a default KMS encryption key will be used for envelope encryption of Kubernetes secrets (AWS managed new KMS key). + * If set to false, and no resouce provider for KMS key is defined (under GlobalResources.KmsKey), then no secrets + * encyrption is applied. + * + * Default is true. + */ + readonly useDefaultSecretEncryption? : boolean = true; + + /** + * GitOps modes to be enabled. If not specified, GitOps mode is not enabled. + */ + readonly enableGitOpsMode?: spi.GitOpsMode; } -export class BlueprintPropsConstraints - implements constraints.ConstraintsType -{ - /** - * id can be no less than 1 character long, and no greater than 63 characters long. - * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ - */ - id = new constraints.StringConstraint(1, 63); - - /** - * name can be no less than 1 character long, and no greater than 63 characters long. - * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ - */ - name = new constraints.StringConstraint(1, 63); +export class BlueprintPropsConstraints implements constraints.ConstraintsType { + /** + * id can be no less than 1 character long, and no greater than 63 characters long. + * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + */ + id = new constraints.StringConstraint(1, 63); + + /** + * name can be no less than 1 character long, and no greater than 63 characters long. + * https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + */ + name = new constraints.StringConstraint(1, 63); } export const enum ControlPlaneLogType { - API = "api", - AUDIT = "audit", - AUTHENTICATOR = "authenticator", - CONTROLLER_MANAGER = "controllerManager", - SCHEDULER = "scheduler", + + API = 'api', + AUDIT = 'audit', + AUTHENTICATOR = 'authenticator', + CONTROLLER_MANAGER = 'controllerManager', + SCHEDULER = 'scheduler' } /** @@ -103,134 +102,103 @@ export const enum ControlPlaneLogType { * in accounts and regions. */ export class BlueprintBuilder implements spi.AsyncStackBuilder { - props: Partial; - env: { - account?: string; - region?: string; - }; - - constructor() { - this.props = { - addOns: new Array(), - teams: new Array(), - resourceProviders: new Map(), - }; - this.env = { - account: process.env.CDK_DEFAULT_ACCOUNT, - region: process.env.CDK_DEFAULT_REGION, - }; - } - - public name(name: string): this { - this.props = { ...this.props, ...{ name } }; - return this; - } - - public account(account?: string): this { - this.env.account = account; - return this; - } - - public region(region?: string): this { - this.env.region = region; - return this; - } - - public version(version: KubernetesVersion): this { - this.props = { ...this.props, ...{ version } }; - return this; - } - - public enableControlPlaneLogTypes(...types: ControlPlaneLogType[]): this { - this.props = { ...this.props, ...{ enableControlPlaneLogTypes: types } }; - return this; - } - - public enableGitOps(mode?: spi.GitOpsMode): this { - this.props = { - ...this.props, - ...{ enableGitOpsMode: mode ?? spi.GitOpsMode.APP_OF_APPS }, + + props: Partial; + env: { + account?: string, + region?: string }; - return this; - } - - public withBlueprintProps(props: Partial): this { - const resourceProviders = this.props.resourceProviders!; - this.props = { ...this.props, ...cloneDeep(props) }; - if (props.resourceProviders) { - this.props.resourceProviders = new Map([ - ...resourceProviders!.entries(), - ...props.resourceProviders.entries(), - ]); + + constructor() { + this.props = { addOns: new Array(), teams: new Array(), resourceProviders: new Map() }; + this.env = { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION + }; } - return this; - } - public addOns(...addOns: spi.ClusterAddOn[]): this { - this.props = { - ...this.props, - ...{ addOns: this.props.addOns?.concat(addOns) }, - }; - return this; - } - - public clusterProvider(clusterProvider: spi.ClusterProvider) { - this.props = { ...this.props, ...{ clusterProvider: clusterProvider } }; - return this; - } - - public id(id: string): this { - this.props = { ...this.props, ...{ id } }; - return this; - } - - public teams(...teams: spi.Team[]): this { - this.props = { - ...this.props, - ...{ teams: this.props.teams?.concat(teams) }, - }; - return this; - } - - public resourceProvider(name: string, provider: spi.ResourceProvider): this { - this.props.resourceProviders?.set(name, provider); - return this; - } - - public useDefaultSecretEncryption(useDefault: boolean): this { - this.props = { - ...this.props, - ...{ useDefaultSecretEncryption: useDefault }, - }; - return this; - } - - public clone(region?: string, account?: string): BlueprintBuilder { - return new BlueprintBuilder() - .withBlueprintProps({ ...this.props }) - .account(account ?? this.env.account) - .region(region ?? this.env.region); - } - - public build( - scope: Construct, - id: string, - stackProps?: cdk.StackProps - ): EksBlueprint { - return new EksBlueprint( - scope, - { ...this.props, ...{ id } }, - { ...{ env: this.env }, ...stackProps } - ); - } - - public async buildAsync( - scope: Construct, - id: string, - stackProps?: cdk.StackProps - ): Promise { - return this.build(scope, id, stackProps).waitForAsyncTasks(); - } + public name(name: string): this { + this.props = { ...this.props, ...{ name } }; + return this; + } + + public account(account?: string): this { + this.env.account = account; + return this; + } + + public region(region?: string): this { + this.env.region = region; + return this; + } + + public version(version: KubernetesVersion): this { + this.props = { ...this.props, ...{ version } }; + return this; + } + + public enableControlPlaneLogTypes(...types: ControlPlaneLogType[]): this { + this.props = { ...this.props, ...{ enableControlPlaneLogTypes: types } }; + return this; + } + + public enableGitOps(mode?: spi.GitOpsMode): this { + this.props = { ...this.props, ...{ enableGitOpsMode: mode ?? spi.GitOpsMode.APP_OF_APPS } }; + return this; + } + + public withBlueprintProps(props: Partial): this { + const resourceProviders = this.props.resourceProviders!; + this.props = { ...this.props, ...cloneDeep(props) }; + if (props.resourceProviders) { + this.props.resourceProviders = new Map([...resourceProviders!.entries(), ...props.resourceProviders.entries()]); + } + return this; + } + + public addOns(...addOns: spi.ClusterAddOn[]): this { + this.props = { ...this.props, ...{ addOns: this.props.addOns?.concat(addOns) } }; + return this; + } + + public clusterProvider(clusterProvider: spi.ClusterProvider) { + this.props = { ...this.props, ...{ clusterProvider: clusterProvider } }; + return this; + } + + public id(id: string): this { + this.props = { ...this.props, ...{ id } }; + return this; + } + + public teams(...teams: spi.Team[]): this { + this.props = { ...this.props, ...{ teams: this.props.teams?.concat(teams) } }; + return this; + } + + public resourceProvider(name: string, provider: spi.ResourceProvider): this { + this.props.resourceProviders?.set(name, provider); + return this; + } + + public useDefaultSecretEncryption(useDefault: boolean): this { + this.props = { ...this.props, ...{ useDefaultSecretEncryption: useDefault } }; + return this; + } + + public clone(region?: string, account?: string): BlueprintBuilder { + return new BlueprintBuilder().withBlueprintProps({ ...this.props }) + .account(account ?? this.env.account).region(region ?? this.env.region); + } + + public build(scope: Construct, id: string, stackProps?: cdk.StackProps): EksBlueprint { + return new EksBlueprint(scope, { ...this.props, ...{ id } }, + { ...{ env: this.env }, ...stackProps }); + } + + public async buildAsync(scope: Construct, id: string, stackProps?: cdk.StackProps): Promise { + return this.build(scope, id, stackProps).waitForAsyncTasks(); + } } /** @@ -238,201 +206,156 @@ export class BlueprintBuilder implements spi.AsyncStackBuilder { * and orchestrates provisioning of add-ons, teams and post deployment hooks. */ export class EksBlueprint extends cdk.Stack { - static readonly USAGE_ID = "qs-1s1r465hk"; - - private asyncTasks: Promise; - - private clusterInfo: spi.ClusterInfo; - - public static builder(): BlueprintBuilder { - return new BlueprintBuilder(); - } - - constructor( - scope: Construct, - blueprintProps: EksBlueprintProps, - props?: cdk.StackProps - ) { - super( - scope, - blueprintProps.id, - utils.withUsageTracking(EksBlueprint.USAGE_ID, props) - ); - this.validateInput(blueprintProps); - - const resourceContext = this.provideNamedResources(blueprintProps); - - let vpcResource: IVpc | undefined = resourceContext.get( - spi.GlobalResources.Vpc - ); - - if (!vpcResource) { - vpcResource = resourceContext.add( - spi.GlobalResources.Vpc, - new VpcProvider() - ); - } - const version = blueprintProps.version ?? KubernetesVersion.V1_24; - let kmsKeyResource: IKey | undefined = resourceContext.get( - spi.GlobalResources.KmsKey - ); + static readonly USAGE_ID = "qs-1s1r465hk"; - if (!kmsKeyResource && blueprintProps.useDefaultSecretEncryption != false) { - kmsKeyResource = resourceContext.add( - spi.GlobalResources.KmsKey, - new CreateKmsKeyProvider() - ); - } + private asyncTasks: Promise; - blueprintProps = this.resolveDynamicProxies( - blueprintProps, - resourceContext - ); - - const clusterProvider = - blueprintProps.clusterProvider ?? - new MngClusterProvider({ - id: `${blueprintProps.name ?? blueprintProps.id}-ng`, - version, - }); - - this.clusterInfo = clusterProvider.createCluster( - this, - vpcResource!, - kmsKeyResource - ); - this.clusterInfo.setResourceContext(resourceContext); - - let enableLogTypes: string[] | undefined = - blueprintProps.enableControlPlaneLogTypes; - if (enableLogTypes) { - utils.setupClusterLogging( - this.clusterInfo.cluster.stack, - this.clusterInfo.cluster, - enableLogTypes - ); + private clusterInfo: spi.ClusterInfo; + + public static builder(): BlueprintBuilder { + return new BlueprintBuilder(); } - if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APPLICATION) { - ArgoGitOpsFactory.enableGitOps(); - } else if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APP_OF_APPS) { - ArgoGitOpsFactory.enableGitOpsAppOfApps(); + constructor(scope: Construct, blueprintProps: EksBlueprintProps, props?: cdk.StackProps) { + super(scope, blueprintProps.id, utils.withUsageTracking(EksBlueprint.USAGE_ID, props)); + this.validateInput(blueprintProps); + + const resourceContext = this.provideNamedResources(blueprintProps); + + let vpcResource: IVpc | undefined = resourceContext.get(spi.GlobalResources.Vpc); + + if (!vpcResource) { + vpcResource = resourceContext.add(spi.GlobalResources.Vpc, new VpcProvider()); + } + + const version = blueprintProps.version ?? KubernetesVersion.V1_24; + let kmsKeyResource: IKey | undefined = resourceContext.get(spi.GlobalResources.KmsKey); + + if (!kmsKeyResource && blueprintProps.useDefaultSecretEncryption != false) { + kmsKeyResource = resourceContext.add(spi.GlobalResources.KmsKey, new CreateKmsKeyProvider()); + } + + blueprintProps = this.resolveDynamicProxies(blueprintProps, resourceContext); + + const clusterProvider = blueprintProps.clusterProvider ?? new MngClusterProvider({ + id: `${blueprintProps.name ?? blueprintProps.id}-ng`, + version + }); + + this.clusterInfo = clusterProvider.createCluster(this, vpcResource!, kmsKeyResource); + this.clusterInfo.setResourceContext(resourceContext); + + let enableLogTypes: string[] | undefined = blueprintProps.enableControlPlaneLogTypes; + if (enableLogTypes) { + utils.setupClusterLogging(this.clusterInfo.cluster.stack, this.clusterInfo.cluster, enableLogTypes); + } + + if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APPLICATION) { + ArgoGitOpsFactory.enableGitOps(); + } else if (blueprintProps.enableGitOpsMode == spi.GitOpsMode.APP_OF_APPS) { + ArgoGitOpsFactory.enableGitOpsAppOfApps(); + } + + const postDeploymentSteps = Array(); + + for (let addOn of (blueprintProps.addOns ?? [])) { // must iterate in the strict order + const result = addOn.deploy(this.clusterInfo); + if (result) { + const addOnKey = utils.getAddOnNameOrId(addOn); + this.clusterInfo.addScheduledAddOn(addOnKey, result, utils.isOrderedAddOn(addOn)); + } + const postDeploy: any = addOn; + if ((postDeploy as spi.ClusterPostDeploy).postDeploy !== undefined) { + postDeploymentSteps.push(postDeploy); + } + } + + const scheduledAddOns = this.clusterInfo.getAllScheduledAddons(); + const addOnKeys = [...scheduledAddOns.keys()]; + const promises = scheduledAddOns.values(); + + this.asyncTasks = Promise.all(promises).then((constructs) => { + constructs.forEach((construct, index) => { + this.clusterInfo.addProvisionedAddOn(addOnKeys[index], construct); + }); + + if (blueprintProps.teams != null) { + for (let team of blueprintProps.teams) { + team.setup(this.clusterInfo); + } + } + + for (let step of postDeploymentSteps) { + step.postDeploy(this.clusterInfo, blueprintProps.teams ?? []); + } + }); + + this.asyncTasks.catch(err => { + console.error(err); + throw new Error(err); + }); } - const postDeploymentSteps = Array(); - - for (let addOn of blueprintProps.addOns ?? []) { - // must iterate in the strict order - const result = addOn.deploy(this.clusterInfo); - if (result) { - const addOnKey = utils.getAddOnNameOrId(addOn); - this.clusterInfo.addScheduledAddOn( - addOnKey, - result, - utils.isOrderedAddOn(addOn) - ); - } - const postDeploy: any = addOn; - if ((postDeploy as spi.ClusterPostDeploy).postDeploy !== undefined) { - postDeploymentSteps.push(postDeploy); - } + /** + * Since constructor cannot be marked as async, adding a separate method to wait + * for async code to finish. + * @returns Promise that resolves to the blueprint + */ + public async waitForAsyncTasks(): Promise { + if (this.asyncTasks) { + return this.asyncTasks.then(() => { + return this; + }); + } + return Promise.resolve(this); } - const scheduledAddOns = this.clusterInfo.getAllScheduledAddons(); - const addOnKeys = [...scheduledAddOns.keys()]; - const promises = scheduledAddOns.values(); + /** + * This method returns all the constructs produced by during the cluster creation (e.g. add-ons). + * May be used in testing for verification. + * @returns cluster info object + */ + getClusterInfo(): spi.ClusterInfo { + return this.clusterInfo; + } - this.asyncTasks = Promise.all(promises).then((constructs) => { - constructs.forEach((construct, index) => { - this.clusterInfo.addProvisionedAddOn(addOnKeys[index], construct); - }); + private provideNamedResources(blueprintProps: EksBlueprintProps): spi.ResourceContext { + const result = new spi.ResourceContext(this, blueprintProps); - if (blueprintProps.teams != null) { - for (let team of blueprintProps.teams) { - team.setup(this.clusterInfo); + for (let [key, value] of blueprintProps.resourceProviders ?? []) { + result.add(key, value); } - } - - for (let step of postDeploymentSteps) { - step.postDeploy(this.clusterInfo, blueprintProps.teams ?? []); - } - }); - - this.asyncTasks.catch((err) => { - console.error(err); - throw new Error(err); - }); - } - - /** - * Since constructor cannot be marked as async, adding a separate method to wait - * for async code to finish. - * @returns Promise that resolves to the blueprint - */ - public async waitForAsyncTasks(): Promise { - if (this.asyncTasks) { - return this.asyncTasks.then(() => { - return this; - }); + + return result; } - return Promise.resolve(this); - } - - /** - * This method returns all the constructs produced by during the cluster creation (e.g. add-ons). - * May be used in testing for verification. - * @returns cluster info object - */ - getClusterInfo(): spi.ClusterInfo { - return this.clusterInfo; - } - - private provideNamedResources( - blueprintProps: EksBlueprintProps - ): spi.ResourceContext { - const result = new spi.ResourceContext(this, blueprintProps); - - for (let [key, value] of blueprintProps.resourceProviders ?? []) { - result.add(key, value); + + /** + * Resolves all dynamic proxies, that substitutes resource provider proxies with the resolved values. + * @param blueprintProps + * @param resourceContext + * @returns a copy of blueprint props with resolved values + */ + private resolveDynamicProxies(blueprintProps: EksBlueprintProps, resourceContext: spi.ResourceContext) : EksBlueprintProps { + return utils.cloneDeep(blueprintProps, (value) => { + return utils.resolveTarget(value, resourceContext); + }); } - return result; - } - - /** - * Resolves all dynamic proxies, that substitutes resource provider proxies with the resolved values. - * @param blueprintProps - * @param resourceContext - * @returns a copy of blueprint props with resolved values - */ - private resolveDynamicProxies( - blueprintProps: EksBlueprintProps, - resourceContext: spi.ResourceContext - ): EksBlueprintProps { - return utils.cloneDeep(blueprintProps, (value) => { - return utils.resolveTarget(value, resourceContext); - }); - } - - /** - * Validates input against basic defined constraints. - * @param blueprintProps - */ - private validateInput(blueprintProps: EksBlueprintProps) { - const teamNames = new Set(); - constraints.validateConstraints( - new BlueprintPropsConstraints(), - EksBlueprintProps.name, - blueprintProps - ); - if (blueprintProps.teams) { - blueprintProps.teams.forEach((e) => { - if (teamNames.has(e.name)) { - throw new Error(`Team ${e.name} is registered more than once`); + /** + * Validates input against basic defined constraints. + * @param blueprintProps + */ + private validateInput(blueprintProps: EksBlueprintProps) { + const teamNames = new Set(); + constraints.validateConstraints(new BlueprintPropsConstraints, EksBlueprintProps.name, blueprintProps); + if (blueprintProps.teams) { + blueprintProps.teams.forEach(e => { + if (teamNames.has(e.name)) { + throw new Error(`Team ${e.name} is registered more than once`); + } + teamNames.add(e.name); + }); } - teamNames.add(e.name); - }); } - } -} +} \ No newline at end of file diff --git a/test/resource-providers/kms-key.test.ts b/test/resource-providers/kms-key.test.ts index 9cb80c763..78ec78ccf 100644 --- a/test/resource-providers/kms-key.test.ts +++ b/test/resource-providers/kms-key.test.ts @@ -1,7 +1,7 @@ import { App } from "aws-cdk-lib"; import * as blueprints from "../../lib"; import { Match, Template } from "aws-cdk-lib/assertions"; -import { CreateKmsKeyProvider } from "../../lib/resource-providers/kms-key"; +import { CreateKmsKeyProvider, LookupKmsKeyProvider } from "../../lib/resource-providers/kms-key"; import { GlobalResources } from "../../lib"; describe("KmsKeyProvider", () => { @@ -78,7 +78,7 @@ describe("KmsKeyProvider", () => { const stack = blueprints.EksBlueprint.builder() .resourceProvider( GlobalResources.KmsKey, - new CreateKmsKeyProvider(undefined, { alias: "any-alias" }) + new LookupKmsKeyProvider("my-custom-eks-key") ) .account("123456789012") .region("us-east-1") @@ -125,21 +125,18 @@ describe("KmsKeyProvider", () => { const template = Template.fromStack(stack); // Then EKS cluster config has no encryption - template.resourcePropertiesCountIs( - "Custom::AWSCDK-EKS-Cluster", - { + template.resourcePropertiesCountIs("Custom::AWSCDK-EKS-Cluster", { Config: { - encryptionConfig: [ + encryptionConfig: [ { - provider: { - keyArn: Match.anyValue(), - }, - resources: ["secrets"], + provider: { + keyArn: Match.anyValue(), + }, + resources: ["secrets"], }, - ], + ], }, - }, - 0 - ); + }, + 0); }); -}); +}); \ No newline at end of file From fd855ae6f7032d2b7f548f858f95e11d2251ded7 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 16:52:05 -0600 Subject: [PATCH 24/33] autoformat: changed lines only --- test/resource-providers/kms-key.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/resource-providers/kms-key.test.ts b/test/resource-providers/kms-key.test.ts index 78ec78ccf..7f95f67ee 100644 --- a/test/resource-providers/kms-key.test.ts +++ b/test/resource-providers/kms-key.test.ts @@ -1,7 +1,7 @@ import { App } from "aws-cdk-lib"; import * as blueprints from "../../lib"; import { Match, Template } from "aws-cdk-lib/assertions"; -import { CreateKmsKeyProvider, LookupKmsKeyProvider } from "../../lib/resource-providers/kms-key"; +import { CreateKmsKeyProvider } from "../../lib/resource-providers/kms-key"; import { GlobalResources } from "../../lib"; describe("KmsKeyProvider", () => { @@ -78,7 +78,7 @@ describe("KmsKeyProvider", () => { const stack = blueprints.EksBlueprint.builder() .resourceProvider( GlobalResources.KmsKey, - new LookupKmsKeyProvider("my-custom-eks-key") + new CreateKmsKeyProvider("my-custom-eks-key") ) .account("123456789012") .region("us-east-1") From 8ea2c689cf0651885553dab4f9fb531914df9ee5 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 17:03:41 -0600 Subject: [PATCH 25/33] Fixing the efs-csi-driver-key alias typo --- examples/blueprint-construct/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 41943e844..47a3f2443 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -146,7 +146,7 @@ export default class BlueprintConstruct { new blueprints.addons.EfsCsiDriverAddOn({ replicaCount: 1, kmsKeys: [ - blueprints.getResource( context => new kms.Key(context.scope, "ebs-csi-driver-key", { alias: "ebs-csi-driver-key"})), + blueprints.getResource( context => new kms.Key(context.scope, "efs-csi-driver-key", { alias: "efs-csi-driver-key"})), ], }), new blueprints.addons.KedaAddOn({ From 17f5bb2bcdd45c71c2d504591ab882525c0055f8 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Thu, 9 Mar 2023 17:07:51 -0600 Subject: [PATCH 26/33] Removing the redundant removalPolicy parameter from the EFS resource provider --- lib/resource-providers/efs.ts | 5 ----- lib/spi/types.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index 946ebedcb..ef1b24a15 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -2,14 +2,12 @@ import { CfnOutput } from "aws-cdk-lib"; import * as ec2 from "aws-cdk-lib/aws-ec2"; import * as efs from "aws-cdk-lib/aws-efs"; import * as kms from "aws-cdk-lib/aws-kms"; -import * as cdk from "aws-cdk-lib/core"; import { GlobalResources, ResourceContext, ResourceProvider } from "../spi"; export interface CreateEfsFileSystemProps { readonly name?: string; readonly efsProps?: Omit; readonly kmsKeyResourceName?: string; - readonly removalPolicy?: cdk.RemovalPolicy; } export interface LookupEfsFileSystemProps { @@ -22,7 +20,6 @@ export interface LookupEfsFileSystemProps { * * @param name The name of the EFS file system to create. * @param efsProps The props used for the file system. - * @param removalPolicy The removal policy to use for the EFS file system. */ export class CreateEfsFileSystemProvider implements ResourceProvider @@ -43,7 +40,6 @@ export class CreateEfsFileSystemProvider if (vpc === undefined) { throw new Error("VPC not found in context"); } - const removalPolicy = this.options.removalPolicy ?? context.removalPolicy; const clusterVpcCidr = vpc.vpcCidrBlock; let kmsKey: kms.IKey | undefined; if (this.options.kmsKeyResourceName) { @@ -70,7 +66,6 @@ export class CreateEfsFileSystemProvider { vpc, securityGroup: efsSG, - removalPolicy, kmsKey, ...this.options.efsProps, } diff --git a/lib/spi/types.ts b/lib/spi/types.ts index 5a2bd125d..7515e41fd 100644 --- a/lib/spi/types.ts +++ b/lib/spi/types.ts @@ -80,7 +80,6 @@ export interface ApplicationRepository extends GitRepositoryReference { export class ResourceContext { private readonly resources: Map = new Map(); - removalPolicy: cdk.RemovalPolicy | undefined; constructor(public readonly scope: cdk.Stack, public readonly blueprintProps: EksBlueprintProps) { } From 82be64935db73c88cdd03bede688a39ef3c5b74d Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 9 Mar 2023 18:00:42 -0600 Subject: [PATCH 27/33] revised per PR --- lib/addons/aws-for-fluent-bit/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/addons/aws-for-fluent-bit/index.ts b/lib/addons/aws-for-fluent-bit/index.ts index 89ed028e8..a8cfa13ab 100644 --- a/lib/addons/aws-for-fluent-bit/index.ts +++ b/lib/addons/aws-for-fluent-bit/index.ts @@ -12,7 +12,12 @@ export interface AwsForFluentBitAddOnProps extends HelmAddOnUserProps { /** * Iam policies for the add-on. */ - iamPolicies?: PolicyStatement[] + iamPolicies?: PolicyStatement[], + + /** + * Create Namespace with the provided one (will not if namespace is kube-system) + */ + createNamespace?: boolean } /** * Default props for the add-on. @@ -24,6 +29,7 @@ const defaultProps: AwsForFluentBitAddOnProps = { version: '0.1.23', repository: 'https://aws.github.io/eks-charts', namespace: 'kube-system', + createNamespace: false, values: {} }; @@ -45,7 +51,12 @@ export class AwsForFluentBitAddOn extends HelmAddOn { deploy(clusterInfo: ClusterInfo): Promise { const cluster = clusterInfo.cluster; - const namespace = this.options.namespace; + const namespace = this.options.namespace!; + + // Create namespace + if (this.options.createNamespace) { + createNamespace(namespace, cluster, true); + } // Create the FluentBut service account. const serviceAccountName = 'aws-for-fluent-bit-sa'; From fb3492382c2ac8d9c9de252c80e4a63d18a1dbd2 Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 9 Mar 2023 18:02:17 -0600 Subject: [PATCH 28/33] removed dependable based on PR review --- lib/addons/jupyterhub/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/addons/jupyterhub/index.ts b/lib/addons/jupyterhub/index.ts index 361cb9b7b..45998665e 100644 --- a/lib/addons/jupyterhub/index.ts +++ b/lib/addons/jupyterhub/index.ts @@ -1,7 +1,7 @@ import * as assert from "assert"; import { Construct } from "constructs"; import { ClusterInfo } from '../../spi'; -import { createNamespace, dependable, setPath } from '../../utils'; +import { createNamespace, setPath } from '../../utils'; import { AwsLoadBalancerControllerAddOn } from "../aws-loadbalancer-controller"; import { HelmAddOn, HelmAddOnProps, HelmAddOnUserProps } from '../helm-addon'; From b4910fca224c3d72ed09cc457ca04a687968f7cd Mon Sep 17 00:00:00 2001 From: shapirov103 Date: Fri, 10 Mar 2023 01:29:42 -0500 Subject: [PATCH 29/33] upgraded core addons for 1.25 support (1.24 as well) --- lib/addons/coredns/index.ts | 2 +- lib/addons/ebs-csi-driver/index.ts | 2 +- lib/addons/kube-proxy/index.ts | 2 +- lib/addons/vpc-cni/index.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/addons/coredns/index.ts b/lib/addons/coredns/index.ts index 1c29a62e9..37a943c61 100644 --- a/lib/addons/coredns/index.ts +++ b/lib/addons/coredns/index.ts @@ -8,7 +8,7 @@ export class CoreDnsAddOn extends CoreAddOn { constructor(version?: string) { super({ addOnName: "coredns", - version: version ?? "v1.8.7-eksbuild.4", + version: version ?? "v1.9.3-eksbuild.2", saName: "coredns" }); } diff --git a/lib/addons/ebs-csi-driver/index.ts b/lib/addons/ebs-csi-driver/index.ts index 6e1bf7def..3050c0328 100644 --- a/lib/addons/ebs-csi-driver/index.ts +++ b/lib/addons/ebs-csi-driver/index.ts @@ -23,7 +23,7 @@ interface EbsCsiDriverAddOnProps { */ const defaultProps = { addOnName: "aws-ebs-csi-driver", - version: "v1.16.0-eksbuild.1", + version: "v1.16.1-eksbuild.1", saName: "ebs-csi-controller-sa", }; diff --git a/lib/addons/kube-proxy/index.ts b/lib/addons/kube-proxy/index.ts index f3b1692e4..9e76e2102 100644 --- a/lib/addons/kube-proxy/index.ts +++ b/lib/addons/kube-proxy/index.ts @@ -8,7 +8,7 @@ export class KubeProxyAddOn extends CoreAddOn { constructor(version?: string) { super({ addOnName: "kube-proxy", - version: version ?? "v1.24.7-eksbuild.2", + version: version ?? "v1.24.10-eksbuild.2", saName: "kube-proxy" }); } diff --git a/lib/addons/vpc-cni/index.ts b/lib/addons/vpc-cni/index.ts index 1a6725907..8ad70d9c9 100644 --- a/lib/addons/vpc-cni/index.ts +++ b/lib/addons/vpc-cni/index.ts @@ -156,7 +156,7 @@ export interface CustomNetworkingConfig { const defaultProps: CoreAddOnProps = { addOnName: 'vpc-cni', - version: 'v1.12.2-eksbuild.1', + version: 'v1.12.5-eksbuild.2', saName: 'vpc-cni', namespace: 'kube-system', controlPlaneAddOn: true, From 793e5bca26258880144ec6223f0af50645292b30 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 10 Mar 2023 08:28:19 -0600 Subject: [PATCH 30/33] adding SA-namespace node dependency --- lib/addons/aws-for-fluent-bit/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/addons/aws-for-fluent-bit/index.ts b/lib/addons/aws-for-fluent-bit/index.ts index a8cfa13ab..0e73199c9 100644 --- a/lib/addons/aws-for-fluent-bit/index.ts +++ b/lib/addons/aws-for-fluent-bit/index.ts @@ -52,11 +52,6 @@ export class AwsForFluentBitAddOn extends HelmAddOn { deploy(clusterInfo: ClusterInfo): Promise { const cluster = clusterInfo.cluster; const namespace = this.options.namespace!; - - // Create namespace - if (this.options.createNamespace) { - createNamespace(namespace, cluster, true); - } // Create the FluentBut service account. const serviceAccountName = 'aws-for-fluent-bit-sa'; @@ -65,6 +60,12 @@ export class AwsForFluentBitAddOn extends HelmAddOn { namespace: namespace }); + // Create namespace + if (this.options.createNamespace) { + const ns = createNamespace(namespace, cluster, true); + sa.node.addDependency(ns); + } + // Apply additional IAM policies to the service account. const policies = this.options.iamPolicies || []; policies.forEach((policy: PolicyStatement) => sa.addToPrincipalPolicy(policy)); From 4057618f791c33324ed54d37d5542ec03af8a6b3 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Mon, 13 Mar 2023 14:18:57 -0500 Subject: [PATCH 31/33] Removing the redundant check and expanding the kmsKeys param's description in the documentation. --- docs/addons/efs-csi-driver.md | 2 +- lib/resource-providers/efs.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/addons/efs-csi-driver.md b/docs/addons/efs-csi-driver.md index 9420c3fd9..2495705e8 100644 --- a/docs/addons/efs-csi-driver.md +++ b/docs/addons/efs-csi-driver.md @@ -35,7 +35,7 @@ const blueprint = blueprints.EksBlueprint.builder() - `version`: Version of the EFS CSI Driver add-on to be installed. Version 2.2.3 will be installed by default if a value is not provided - `replicaCount`: Number of replicas to be deployed. If not provided, two replicas will be deployed. Note that the number of replicas should be less than or equal to the number of nodes in the cluster otherwise some pods will be left of pending state -- `kmsKeys`: List of KMS keys to be used for encryption-at-rest +- `kmsKeys`: List of KMS keys used for encryption-at-rest, so that the IAM policy can be updated to allow the EFS CSI driver to access the keys ## Validation diff --git a/lib/resource-providers/efs.ts b/lib/resource-providers/efs.ts index ef1b24a15..4f640f100 100644 --- a/lib/resource-providers/efs.ts +++ b/lib/resource-providers/efs.ts @@ -112,9 +112,6 @@ export class LookupEfsFileSystemProvider } ); - if (!efsFileSystem) { - throw new Error("EFS file system not found"); - } new CfnOutput(context.scope, "EfsFileSystemId", { value: efsFileSystem.fileSystemId, }); From ff2d177d124d2245b97b4364764accc51f7b0cca Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Mon, 13 Mar 2023 19:47:25 -0500 Subject: [PATCH 32/33] Fixing some comments and docs to use CreateKmsKeyProvider instead of KmsKeyProvider --- docs/resource-providers/index.md | 2 +- lib/resource-providers/kms-key.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/resource-providers/index.md b/docs/resource-providers/index.md index 1e394ddd2..54dba254a 100644 --- a/docs/resource-providers/index.md +++ b/docs/resource-providers/index.md @@ -132,7 +132,7 @@ blueprints.EksBlueprint.builder() // Specify VPC for the cluster (if not set, a new VPC will be provisioned as per EKS Best Practices) .resourceProvider(GlobalResources.VPC, new VpcProvider(myVpcId) // Specify KMS Key as cluster secrets encryption key - .resourceProvider(GlobalResources.KmsKey, new KmsKeyProvider('my-alias-name') + .resourceProvider(GlobalResources.KmsKey, new CreateKmsKeyProvider('my-alias-name') // Register hosted zone and give it a name of GlobalResources.HostedZone .resourceProvider(GlobalResources.HostedZone, new ImportHostedZoneProvider('hosted-zone-id1', 'my.domain.com')) .resourceProvider("internal-hosted-zone", new ImportHostedZoneProvider('hosted-zone-id2', 'myinternal.domain.com')) diff --git a/lib/resource-providers/kms-key.ts b/lib/resource-providers/kms-key.ts index ec4677b10..9a9beb84b 100644 --- a/lib/resource-providers/kms-key.ts +++ b/lib/resource-providers/kms-key.ts @@ -7,7 +7,7 @@ import { ResourceContext, ResourceProvider } from "../spi"; * @example * ```typescript * const stack = blueprints.EksBlueprint.builder() - * .resourceProvider(GlobalResources.KmsKey, new KmsKeyProvider("my-custom-eks-key")) + * .resourceProvider(GlobalResources.KmsKey, new CreateKmsKeyProvider("my-custom-eks-key")) * .account("123456789012") * .region("us-east-1") * .build(app, "east-test-1"); From 3583361a1ee1107396b53a3451b172ffd7852fd8 Mon Sep 17 00:00:00 2001 From: Aliaksei Ivanou Date: Mon, 13 Mar 2023 19:52:54 -0500 Subject: [PATCH 33/33] Fixed tests --- test/resource-providers/resource-proxy.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/resource-providers/resource-proxy.test.ts b/test/resource-providers/resource-proxy.test.ts index 3d8a0f4e1..ca5a6f959 100644 --- a/test/resource-providers/resource-proxy.test.ts +++ b/test/resource-providers/resource-proxy.test.ts @@ -6,7 +6,7 @@ import { Role, ServicePrincipal } from "aws-cdk-lib/aws-iam"; import { Key } from "aws-cdk-lib/aws-kms"; import * as nutil from 'node:util/types'; import * as blueprints from "../../lib"; -import { AppMeshAddOn, EksBlueprint, GlobalResources, KmsKeyProvider } from "../../lib"; +import { AppMeshAddOn, EksBlueprint, GlobalResources, CreateKmsKeyProvider } from "../../lib"; import { cloneDeep, logger } from "../../lib/utils"; @@ -98,7 +98,7 @@ describe("ResourceProxy",() => { const someRole: Role = blueprints.getResource( context => new Role(context.scope, "some-role", { assumedBy: new ServicePrincipal("sns.amazon.com")})); const builder = EksBlueprint.builder() - .resourceProvider(GlobalResources.KmsKey, new KmsKeyProvider()) + .resourceProvider(GlobalResources.KmsKey, new CreateKmsKeyProvider()) .account("123456789012") .region("us-east-1") .addOns(new AppMeshAddOn( {