From 3423025fd353546da96704bf09589fb8d6fb259e Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 16 Jul 2023 11:16:37 -0500 Subject: [PATCH 01/10] first commit - adding the addons, manifests, and example with GPU nodes --- examples/blueprint-construct/index.ts | 21 +++- lib/addons/index.ts | 1 + lib/addons/neuron/index.ts | 31 ++++++ .../neuron/k8s-neuron-device-plugin-rbac.ytpl | 58 +++++++++++ .../neuron/k8s-neuron-device-plugin.ytpl | 97 +++++++++++++++++++ 5 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 lib/addons/neuron/index.ts create mode 100644 lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl create mode 100644 lib/addons/neuron/k8s-neuron-device-plugin.ytpl diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 3f9ec2491..6483327a4 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -182,6 +182,7 @@ export default class BlueprintConstruct { efsFileSystem: 'apache-airflow-efs-provider' }), new blueprints.ExternalsSecretsAddOn(), + new blueprints.NeuronPluginAddOn(), ]; // Instantiated to for helm version check. @@ -201,7 +202,8 @@ export default class BlueprintConstruct { managedNodeGroups: [ addGenericNodeGroup(), addCustomNodeGroup(), - addWindowsNodeGroup() + addWindowsNodeGroup(), + addInferentiaGroup(), ] }); @@ -345,6 +347,21 @@ function addWindowsNodeGroup(): blueprints.ManagedNodeGroup { }; } - +function addInferentiaGroup(): blueprints.ManagedNodeGroup { + return { + id: "mng4-inferentia", + amiType: NodegroupAmiType.AL2_X86_64_GPU, + instanceTypes: [new ec2.InstanceType('inf1.2xlarge')], + desiredSize: 1, + minSize: 1, + nodeRole: blueprints.getNamedResource("node-role") as iam.Role, + diskSize: 50, + tags:{ + "Name": "Mng4", + "Type": "Inferentia-Node-Group", + "kubernetes.io/cluster/blueprint-construct-dev": "owned" + } + }; +} diff --git a/lib/addons/index.ts b/lib/addons/index.ts index c085e2f93..eb7f83d0b 100644 --- a/lib/addons/index.ts +++ b/lib/addons/index.ts @@ -52,6 +52,7 @@ export * from './emr-on-eks'; export * from './aws-batch-on-eks'; export * from './upbound-universal-crossplane'; export * from './apache-airflow'; +export * from './neuron'; export class Constants { public static readonly BLUEPRINTS_ADDON = "blueprints-addon"; diff --git a/lib/addons/neuron/index.ts b/lib/addons/neuron/index.ts new file mode 100644 index 000000000..30d0a63d2 --- /dev/null +++ b/lib/addons/neuron/index.ts @@ -0,0 +1,31 @@ +import { KubernetesManifest } from "aws-cdk-lib/aws-eks"; +import { ClusterAddOn, ClusterInfo } from "../../spi"; +import { loadYaml, readYamlDocument } from "../../utils/yaml-utils"; + +export class NeuronPluginAddOn implements ClusterAddOn { + deploy(clusterInfo: ClusterInfo): void { + const cluster = clusterInfo.cluster; + + // Read in YAML docs + const rbac = readYamlDocument(__dirname + '/k8s-neuron-device-plugin-rbac.ytpl'); + const plugin = readYamlDocument(__dirname + '/k8s-neuron-device-plugin.ytpl'); + + // Apply Manifests + const rbacLoadYaml = rbac.split("---").map(e => loadYaml(e)); + const rbacManifest = new KubernetesManifest(cluster.stack, "neuron-rbac-manifest", { + cluster, + manifest: rbacLoadYaml, + overwrite: true + }); + + const pluginLoadYaml = plugin.split("---").map(e => loadYaml(e)); + const pluginManifest = new KubernetesManifest(cluster.stack, "neuron-plugin-manifest", { + cluster, + manifest: pluginLoadYaml, + overwrite: true + }); + + // Plugin dependency on the RBAC manifest + pluginManifest.node.addDependency(rbacManifest); + } +} \ No newline at end of file diff --git a/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl b/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl new file mode 100644 index 000000000..28bf10209 --- /dev/null +++ b/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl @@ -0,0 +1,58 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: neuron-device-plugin +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - update + - patch + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - update +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: neuron-device-plugin + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: neuron-device-plugin + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: neuron-device-plugin +subjects: +- kind: ServiceAccount + name: neuron-device-plugin + namespace: kube-system diff --git a/lib/addons/neuron/k8s-neuron-device-plugin.ytpl b/lib/addons/neuron/k8s-neuron-device-plugin.ytpl new file mode 100644 index 000000000..2c909d040 --- /dev/null +++ b/lib/addons/neuron/k8s-neuron-device-plugin.ytpl @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: neuron-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: neuron-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: neuron-device-plugin-ds + spec: + serviceAccount: neuron-device-plugin + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: aws.amazon.com/neuron + operator: Exists + effect: NoSchedule + # Mark this pod as a critical add-on; when enabled, the critical add-on + # scheduler reserves resources for critical add-on pods so that they can + # be rescheduled after a failure. + # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + priorityClassName: "system-node-critical" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "beta.kubernetes.io/instance-type" + operator: In + values: + - inf1.xlarge + - inf1.2xlarge + - inf1.6xlarge + - inf1.24xlarge + - inf2.xlarge + - inf2.4xlarge + - inf2.8xlarge + - inf2.24xlarge + - inf2.48xlarge + - trn1.2xlarge + - trn1.32xlarge + - trn1n.32xlarge + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - inf1.xlarge + - inf1.2xlarge + - inf1.6xlarge + - inf1.24xlarge + - inf2.xlarge + - inf2.4xlarge + - inf2.8xlarge + - inf2.24xlarge + - inf2.48xlarge + - trn1.2xlarge + - trn1.32xlarge + - trn1n.32xlarge + containers: + #Device Plugin containers are available both in us-east and us-west ecr + #repos + - image: public.ecr.aws/neuron/neuron-device-plugin:2.12.5.0 + imagePullPolicy: Always + name: neuron-device-plugin + env: + - name: KUBECONFIG + value: /etc/kubernetes/kubelet.conf + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: infa-map + mountPath: /run + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: infa-map + hostPath: + path: /run + + From 25a717bcc3c1a61ca2874855018c22225879f402 Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 16 Jul 2023 11:53:40 -0500 Subject: [PATCH 02/10] add documentations --- docs/addons/neuron-plugin-addon.md | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/addons/neuron-plugin-addon.md diff --git a/docs/addons/neuron-plugin-addon.md b/docs/addons/neuron-plugin-addon.md new file mode 100644 index 000000000..c0cb007d0 --- /dev/null +++ b/docs/addons/neuron-plugin-addon.md @@ -0,0 +1,49 @@ +# Neuron Device Plugin Addon + +[AWS Neuron](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/) is the SDK used to run deep learning workloads on AWS Inferentia and AWS Trainium based instances. This addon will install the Neuron Device Plugin necessary to run the instances on Amazon EKS (and Blueprints). Note that you **must** use a GPU nodegroup with *inf1, inf2, trn1,* or *trn1n* instances. + +## Usage + +#### **`index.ts`** +```typescript +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import * as blueprints from '@aws-quickstart/eks-blueprints'; + +const app = new cdk.App(); + +const addOn = new blueprints.addons.NeuronPluginAddon(); + +const clusterProvider = new blueprints.GenericClusterProvider({ + version: KubernetesVersion.V1_25, + managedNodeGroups: [ + new ManagedNodeGroup({ + id: "mng1", + amiType: NodegroupAmiType.AL2_X86_64_GPU, + instanceTypes: [new ec2.InstanceType('inf1.2xlarge')], + desiredSize: 1, + maxSize: 2, + nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, + }) + ] +}); + +const blueprint = blueprints.EksBlueprint.builder() + .clusterProvider(clusterProvider) + .addOns(addOn) + .build(app, 'my-stack-name'); +``` + +Once deployed, you can see the plugin daemonset in the `kube-system` namespace. + +```sh +$ kubectl get daemonset neuron-device-plugin-daemonset -n kube-system + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +neuron-device-plugin-daemonset 1 1 1 1 1 24m 20m +``` + +## Functionality + +1. Deploys the plugin daemonset in `kube-system` namespace by default. +2. Provides a plugin for the blueprint to leverage the Inferentia or Trainium instances to use the Neuron SDK. \ No newline at end of file From 402eb101ac9bec1168a78c209b996fba79a1a892 Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 16 Jul 2023 17:23:35 -0500 Subject: [PATCH 03/10] slight fix to the docs --- docs/addons/neuron-plugin-addon.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/addons/neuron-plugin-addon.md b/docs/addons/neuron-plugin-addon.md index c0cb007d0..aa24271fb 100644 --- a/docs/addons/neuron-plugin-addon.md +++ b/docs/addons/neuron-plugin-addon.md @@ -17,16 +17,20 @@ const addOn = new blueprints.addons.NeuronPluginAddon(); const clusterProvider = new blueprints.GenericClusterProvider({ version: KubernetesVersion.V1_25, managedNodeGroups: [ - new ManagedNodeGroup({ - id: "mng1", + inferentiaNodeGroup() + ] +}); + +function inferentiaNodeGroup(): blueprints.ManagedNodeGroup { + return { + id: "mng1", amiType: NodegroupAmiType.AL2_X86_64_GPU, instanceTypes: [new ec2.InstanceType('inf1.2xlarge')], desiredSize: 1, maxSize: 2, nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, - }) - ] -}); + }; +} const blueprint = blueprints.EksBlueprint.builder() .clusterProvider(clusterProvider) From 38b70f6d5fdf00ff8ba28e038b2262d9ff219bae Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 12 Oct 2023 18:24:06 -0500 Subject: [PATCH 04/10] test fix, lint fix, and removed local testing example blueprint --- examples/blueprint-construct/index.ts | 18 +++ examples/young-construct/index.ts | 154 -------------------------- lib/addons/neuron/index.ts | 47 ++++---- lib/utils/yaml-utils.ts | 2 +- test/utils/yaml-utils.test.ts | 63 ++++++++++- 5 files changed, 104 insertions(+), 180 deletions(-) delete mode 100644 examples/young-construct/index.ts diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 034854eb3..57195ae86 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -233,6 +233,7 @@ export default class BlueprintConstruct { addGenericNodeGroup(), addCustomNodeGroup(), addWindowsNodeGroup(), + addInferentiaNodeGroup(), ] }); @@ -376,6 +377,23 @@ function addWindowsNodeGroup(): blueprints.ManagedNodeGroup { }; } +function addInferentiaNodeGroup(): blueprints.ManagedNodeGroup { + + return { + id: "mng4-inferentia", + instanceTypes: [new ec2.InstanceType('inf1.2xlarge')], + desiredSize: 1, + minSize: 1, + nodeRole: blueprints.getNamedResource("node-role") as iam.Role, + diskSize: 50, + tags: { + "Name": "Mng4", + "Type": "Managed-InferentiaNode-Group", + "LaunchTemplate": "Inferentia", + "kubernetes.io/cluster/blueprint-construct-dev": "owned" + } + }; +} diff --git a/examples/young-construct/index.ts b/examples/young-construct/index.ts deleted file mode 100644 index 996a49217..000000000 --- a/examples/young-construct/index.ts +++ /dev/null @@ -1,154 +0,0 @@ -import * as cdk from 'aws-cdk-lib'; -import * as ec2 from "aws-cdk-lib/aws-ec2"; -import { KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; -import { AccountRootPrincipal, Role } from 'aws-cdk-lib/aws-iam'; -import { Construct } from "constructs"; -import * as blueprints from '../../lib'; -import { logger, userLog } from '../../lib/utils'; - -export interface BlueprintConstructProps { - /** - * Id - */ - id: string -} - -export default class YoungConstruct { - constructor(scope: Construct, props: cdk.StackProps) { - - blueprints.HelmAddOn.validateHelmVersions = true; - blueprints.HelmAddOn.failOnVersionValidation = false; - logger.settings.minLevel = 3; - userLog.settings.minLevel = 2; - - const vpc = new blueprints.VpcProvider(undefined, { - primaryCidr: "10.2.0.0/16", - secondaryCidr: "100.64.0.0/16", - secondarySubnetCidrs: ["100.64.0.0/24","100.64.1.0/24","100.64.2.0/24"] - }); - // const airflowEfs = new blueprints.CreateEfsFileSystemProvider({ - // name: "airflow-efs-file-system" - // }); - const apacheAirflowS3Bucket = new blueprints.CreateS3BucketProvider({ - id: 'apache-airflow-s3-bucket-id', - s3BucketProps: { removalPolicy: cdk.RemovalPolicy.DESTROY } - }); - - const teams: Array = []; - const addOns: Array = [ - new blueprints.addons.AwsLoadBalancerControllerAddOn(), - new blueprints.addons.CoreDnsAddOn(), - new blueprints.addons.KubeProxyAddOn(), - new blueprints.addons.SSMAgentAddOn(), - // new blueprints.addons.KarpenterAddOn({ - // requirements: [ - // { key: 'node.kubernetes.io/instance-type', op: 'In', vals: ['m5.2xlarge'] }, - // { key: 'topology.kubernetes.io/zone', op: 'NotIn', vals: ['us-west-2c']}, - // { key: 'kubernetes.io/arch', op: 'In', vals: ['amd64','arm64']}, - // { key: 'karpenter.sh/capacity-type', op: 'In', vals: ['spot']}, - // ], - // subnetTags: { - // "Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", - // }, - // securityGroupTags: { - // "kubernetes.io/cluster/blueprint-construct-dev": "owned", - // }, - // taints: [{ - // key: "workload", - // value: "test", - // effect: "NoSchedule", - // }], - // consolidation: { enabled: true }, - // ttlSecondsUntilExpired: 2592000, - // weight: 20, - // interruptionHandling: true, - // limits: { - // resources: { - // cpu: 20, - // memory: "64Gi", - // } - // }, - // }), - new blueprints.addons.EfsCsiDriverAddOn({replicaCount: 1}), - new blueprints.addons.EbsCsiDriverAddOn(), - // new blueprints.addons.JupyterHubAddOn({ - // efsConfig: { - // pvcName: "efs-persist", - // removalPolicy: cdk.RemovalPolicy.DESTROY, - // capacity: '10Gi', - // }, - // serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, - // notebookStack: 'jupyter/datascience-notebook', - // values: { prePuller: { hook: { enabled: false }}} - // }), - // new blueprints.AwsBatchAddOn(), - new blueprints.AwsForFluentBitAddOn(), - // new blueprints.AirflowAddOn({ - // enableLogging: true, - // s3BucketName: airflowS3bucket.name - // // enableRds: true, - // // dbConfig: { - // // username: "airflow-user", - // // password: "PA$$w0rd123", - // // dbName: "airflow", - // // }, - // // enableEfs: true, - // // efsFileSystemName: airflowEfs.options.name!, - // }), - new blueprints.ApacheAirflowAddOn({ - enableLogging: true, - s3Bucket: 'airflow-logging-s3-bucket', - // enableEfs: true, - // efsFileSystem: 'apache-airflow-efs-provider', - }) - ]; - - const blueprintID = 'young-blueprint-test'; - - const clusterProvider = new blueprints.GenericClusterProvider({ - version: KubernetesVersion.V1_25, - mastersRole: blueprints.getResource(context => { - return new Role(context.scope, 'AdminRole', { assumedBy: new AccountRootPrincipal() }); - }), - managedNodeGroups: [ - { - id: "mng1", - amiType: NodegroupAmiType.AL2_X86_64, - // amiReleaseVersion: "", - instanceTypes: [new ec2.InstanceType('m5.4xlarge')], - diskSize: 25, - desiredSize: 2, - maxSize: 3, - nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS } - } - ] - }); - - // const batchTeam: blueprints.BatchEksTeamProps = { - // name: 'batch-a', - // namespace: 'aws-batch', - // envName: 'batch-a-comp-env', - // computeResources: { - // envType: blueprints.BatchEnvType.EC2, - // allocationStrategy: blueprints.BatchAllocationStrategy.BEST, - // priority: 10, - // minvCpus: 0, - // maxvCpus: 128, - // instanceTypes: ["m5", "c4.4xlarge"] - // }, - // jobQueueName: 'team-a-job-queue', - // }; - - blueprints.EksBlueprint.builder() - .resourceProvider(blueprints.GlobalResources.Vpc, vpc) - .resourceProvider('airflow-logging-s3-bucket', apacheAirflowS3Bucket) - // .resourceProvider('airflow-efs-file-system', airflowEfs) - .addOns(...addOns) - .clusterProvider(clusterProvider) - .teams(...teams, - // new blueprints.BatchEksTeam(batchTeam) - ) - // .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) - .build(scope, blueprintID, props); - } -} \ No newline at end of file diff --git a/lib/addons/neuron/index.ts b/lib/addons/neuron/index.ts index 6ded8cb0e..dc05f2c29 100644 --- a/lib/addons/neuron/index.ts +++ b/lib/addons/neuron/index.ts @@ -1,40 +1,39 @@ import { Construct } from "constructs"; -import { KubernetesManifest } from "aws-cdk-lib/aws-eks"; import { ClusterAddOn, ClusterInfo } from "../../spi"; import { KubectlProvider, ManifestDeployment } from "../helm-addon/kubectl-provider"; -import { loadYaml, loadExternalYaml } from "../../utils/yaml-utils"; +import { loadMultiResourceExternalYaml } from "../../utils/yaml-utils"; -const PLUGIN_URL = "https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-device-plugin.yml" -const RBAC_URL = "https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-device-plugin-rbac.yml" +const PLUGIN_URL = "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/master/src/k8/k8s-neuron-device-plugin.yml"; +const RBAC_URL = "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/master/src/k8/k8s-neuron-device-plugin-rbac.yml"; export class NeuronPluginAddOn implements ClusterAddOn { deploy(clusterInfo: ClusterInfo): Promise { - const cluster = clusterInfo.cluster; const kubectlProvider = new KubectlProvider(clusterInfo); // Read in YAML docs - const rbac = loadExternalYaml(RBAC_URL); - const plugin = loadExternalYaml(PLUGIN_URL); - - // Apply Manifests - const rbacLoadYaml = rbac.split("---").map((e: any) => loadYaml(e)); - const rbacManifest = new KubernetesManifest(cluster.stack, "neuron-rbac-manifest", { - cluster, - manifest: rbacLoadYaml, - overwrite: true - }); - - const pluginLoadYaml = plugin.split("---").map((e: any) => loadYaml(e)); - const pluginManifest = new KubernetesManifest(cluster.stack, "neuron-plugin-manifest", { - cluster, - manifest: pluginLoadYaml, - overwrite: true - }); + const rbac = loadMultiResourceExternalYaml(RBAC_URL); + const rbacManifest: ManifestDeployment = { + name: "neuron-rbac-manifest", + namespace: "", + manifest: rbac, + values: {} + }; + + const plugin = loadMultiResourceExternalYaml(PLUGIN_URL); + const pluginManifest: ManifestDeployment = { + name: "neuron-plugin-manifest", + namespace: "kube-system", + manifest: plugin, + values: {} + }; + + const rbacStatement = kubectlProvider.addManifest(rbacManifest); + const pluginStatement = kubectlProvider.addManifest(pluginManifest); // Plugin dependency on the RBAC manifest - pluginManifest.node.addDependency(rbacManifest); + pluginStatement.node.addDependency(rbacStatement); - return Promise.resolve(pluginManifest); + return Promise.resolve(pluginStatement); } } \ No newline at end of file diff --git a/lib/utils/yaml-utils.ts b/lib/utils/yaml-utils.ts index ad7ed1550..e6059b687 100644 --- a/lib/utils/yaml-utils.ts +++ b/lib/utils/yaml-utils.ts @@ -42,7 +42,7 @@ export function loadMultiResourceYaml(path: string): any { export function loadMultiResourceExternalYaml(url: string): any { const doc = loadExternalYaml(url); - return doc.split("---").map((e: any) => loadYaml(e)); + return doc; } export function loadYaml(document: string): any { diff --git a/test/utils/yaml-utils.test.ts b/test/utils/yaml-utils.test.ts index e91c94422..32a51cf23 100644 --- a/test/utils/yaml-utils.test.ts +++ b/test/utils/yaml-utils.test.ts @@ -13,7 +13,7 @@ describe('Unit tests for yaml utils', () => { const serialized = yaml.serializeYaml(sample); - expect(serialized.length).toBe(3); + expect(serialized.length).toBe(41); }); test("The YAML Document with multiple resources is read correctly", () => { @@ -28,4 +28,65 @@ describe('Unit tests for yaml utils', () => { expect(doc[2]).toStrictEqual(secondPart); expect(doc[3]).toStrictEqual(lastPart); }); + + test("External YAML Document is read correctly", () => { + const doc = yaml.loadExternalYaml('https://raw.githubusercontent.com/kubernetes/examples/master/guestbook/legacy/frontend-controller.yaml'); + const part = { + apiVersion: "v1", + kind: "ReplicationController", + metadata: {name: "frontend"}, + spec: { + replicas: 3, + template: { + metadata: { + labels: {app: "guestbook", tier: "frontend"} + }, + spec: { + containers: [{ + name: "php-redis", + image: "gcr.io/google_samples/gb-frontend:v4", + resources: { + requests: { + cpu: "100m", + memory: "100Mi" + } + }, + env: [{name: "GET_HOSTS_FROM", value: "dns"}], + ports:[{containerPort: 80}] + }] + } + } + } + }; + + expect(doc.length).toBe(1); + expect(doc[0]).toStrictEqual(part); + }); + + test("External YAML Document with multiple resources is read correctly", () => { + const doc = yaml.loadMultiResourceExternalYaml('https://raw.githubusercontent.com/kubernetes/examples/master/guestbook/all-in-one/frontend.yaml'); + const firstPart = { + apiVersion: "v1", + kind: "Service", + metadata: { + name: "frontend", + labels:{ + app: "guestbook", + tier: "frontend" + } + }, + spec: + { + type: "NodePort", + ports: [{port: 80}], + selector: { + app: "guestbook", + tier: "frontend" + } + } + }; + + expect(doc.length).toBe(2); + expect(doc[0]).toStrictEqual(firstPart); + }); }); From 56432405403489035b133aa02fad4728c840d6d1 Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 12 Oct 2023 20:51:36 -0500 Subject: [PATCH 05/10] doc fix to remove nodegroup --- docs/addons/neuron-plugin-addon.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/addons/neuron-plugin-addon.md b/docs/addons/neuron-plugin-addon.md index aa24271fb..445170c4b 100644 --- a/docs/addons/neuron-plugin-addon.md +++ b/docs/addons/neuron-plugin-addon.md @@ -1,6 +1,6 @@ # Neuron Device Plugin Addon -[AWS Neuron](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/) is the SDK used to run deep learning workloads on AWS Inferentia and AWS Trainium based instances. This addon will install the Neuron Device Plugin necessary to run the instances on Amazon EKS (and Blueprints). Note that you **must** use a GPU nodegroup with *inf1, inf2, trn1,* or *trn1n* instances. +[AWS Neuron](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/) is the SDK used to run deep learning workloads on AWS Inferentia and AWS Trainium based instances. This addon will install the Neuron Device Plugin necessary to run the instances on Amazon EKS (and Blueprints). Note that you **must** use *inf1, inf2, trn1,* or *trn1n* instances. ## Usage @@ -15,7 +15,7 @@ const app = new cdk.App(); const addOn = new blueprints.addons.NeuronPluginAddon(); const clusterProvider = new blueprints.GenericClusterProvider({ - version: KubernetesVersion.V1_25, + version: KubernetesVersion.V1_27, managedNodeGroups: [ inferentiaNodeGroup() ] @@ -24,7 +24,6 @@ const clusterProvider = new blueprints.GenericClusterProvider({ function inferentiaNodeGroup(): blueprints.ManagedNodeGroup { return { id: "mng1", - amiType: NodegroupAmiType.AL2_X86_64_GPU, instanceTypes: [new ec2.InstanceType('inf1.2xlarge')], desiredSize: 1, maxSize: 2, From 7f1302a6ff404a2ae42ed1f6f6c6d8394fb3d447 Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 12 Oct 2023 21:28:52 -0500 Subject: [PATCH 06/10] added back GPU node group --- examples/blueprint-construct/index.ts | 24 +++- examples/young-construct/index.ts | 154 ++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 examples/young-construct/index.ts diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 57195ae86..f8ba490d3 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -232,7 +232,8 @@ export default class BlueprintConstruct { managedNodeGroups: [ addGenericNodeGroup(), addCustomNodeGroup(), - addWindowsNodeGroup(), + addWindowsNodeGroup(), // commented out to check the impact on e2e + addGpuNodeGroup(), addInferentiaNodeGroup(), ] }); @@ -377,6 +378,27 @@ function addWindowsNodeGroup(): blueprints.ManagedNodeGroup { }; } +function addGpuNodeGroup(): blueprints.ManagedNodeGroup { + + return { + id: "mng-linux-gpu", + amiType: NodegroupAmiType.AL2_X86_64_GPU, + instanceTypes: [new ec2.InstanceType('g5.xlarge')], + desiredSize: 0, + minSize: 0, + maxSize: 1, + nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, + launchTemplate: { + tags: { + "Name": "Mng-linux-Gpu", + "Type": "Managed-linux-Gpu-Node-Group", + "LaunchTemplate": "Linux-Launch-Template", + }, + requireImdsv2: false + } + }; +} + function addInferentiaNodeGroup(): blueprints.ManagedNodeGroup { return { diff --git a/examples/young-construct/index.ts b/examples/young-construct/index.ts new file mode 100644 index 000000000..996a49217 --- /dev/null +++ b/examples/young-construct/index.ts @@ -0,0 +1,154 @@ +import * as cdk from 'aws-cdk-lib'; +import * as ec2 from "aws-cdk-lib/aws-ec2"; +import { KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; +import { AccountRootPrincipal, Role } from 'aws-cdk-lib/aws-iam'; +import { Construct } from "constructs"; +import * as blueprints from '../../lib'; +import { logger, userLog } from '../../lib/utils'; + +export interface BlueprintConstructProps { + /** + * Id + */ + id: string +} + +export default class YoungConstruct { + constructor(scope: Construct, props: cdk.StackProps) { + + blueprints.HelmAddOn.validateHelmVersions = true; + blueprints.HelmAddOn.failOnVersionValidation = false; + logger.settings.minLevel = 3; + userLog.settings.minLevel = 2; + + const vpc = new blueprints.VpcProvider(undefined, { + primaryCidr: "10.2.0.0/16", + secondaryCidr: "100.64.0.0/16", + secondarySubnetCidrs: ["100.64.0.0/24","100.64.1.0/24","100.64.2.0/24"] + }); + // const airflowEfs = new blueprints.CreateEfsFileSystemProvider({ + // name: "airflow-efs-file-system" + // }); + const apacheAirflowS3Bucket = new blueprints.CreateS3BucketProvider({ + id: 'apache-airflow-s3-bucket-id', + s3BucketProps: { removalPolicy: cdk.RemovalPolicy.DESTROY } + }); + + const teams: Array = []; + const addOns: Array = [ + new blueprints.addons.AwsLoadBalancerControllerAddOn(), + new blueprints.addons.CoreDnsAddOn(), + new blueprints.addons.KubeProxyAddOn(), + new blueprints.addons.SSMAgentAddOn(), + // new blueprints.addons.KarpenterAddOn({ + // requirements: [ + // { key: 'node.kubernetes.io/instance-type', op: 'In', vals: ['m5.2xlarge'] }, + // { key: 'topology.kubernetes.io/zone', op: 'NotIn', vals: ['us-west-2c']}, + // { key: 'kubernetes.io/arch', op: 'In', vals: ['amd64','arm64']}, + // { key: 'karpenter.sh/capacity-type', op: 'In', vals: ['spot']}, + // ], + // subnetTags: { + // "Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", + // }, + // securityGroupTags: { + // "kubernetes.io/cluster/blueprint-construct-dev": "owned", + // }, + // taints: [{ + // key: "workload", + // value: "test", + // effect: "NoSchedule", + // }], + // consolidation: { enabled: true }, + // ttlSecondsUntilExpired: 2592000, + // weight: 20, + // interruptionHandling: true, + // limits: { + // resources: { + // cpu: 20, + // memory: "64Gi", + // } + // }, + // }), + new blueprints.addons.EfsCsiDriverAddOn({replicaCount: 1}), + new blueprints.addons.EbsCsiDriverAddOn(), + // new blueprints.addons.JupyterHubAddOn({ + // efsConfig: { + // pvcName: "efs-persist", + // removalPolicy: cdk.RemovalPolicy.DESTROY, + // capacity: '10Gi', + // }, + // serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, + // notebookStack: 'jupyter/datascience-notebook', + // values: { prePuller: { hook: { enabled: false }}} + // }), + // new blueprints.AwsBatchAddOn(), + new blueprints.AwsForFluentBitAddOn(), + // new blueprints.AirflowAddOn({ + // enableLogging: true, + // s3BucketName: airflowS3bucket.name + // // enableRds: true, + // // dbConfig: { + // // username: "airflow-user", + // // password: "PA$$w0rd123", + // // dbName: "airflow", + // // }, + // // enableEfs: true, + // // efsFileSystemName: airflowEfs.options.name!, + // }), + new blueprints.ApacheAirflowAddOn({ + enableLogging: true, + s3Bucket: 'airflow-logging-s3-bucket', + // enableEfs: true, + // efsFileSystem: 'apache-airflow-efs-provider', + }) + ]; + + const blueprintID = 'young-blueprint-test'; + + const clusterProvider = new blueprints.GenericClusterProvider({ + version: KubernetesVersion.V1_25, + mastersRole: blueprints.getResource(context => { + return new Role(context.scope, 'AdminRole', { assumedBy: new AccountRootPrincipal() }); + }), + managedNodeGroups: [ + { + id: "mng1", + amiType: NodegroupAmiType.AL2_X86_64, + // amiReleaseVersion: "", + instanceTypes: [new ec2.InstanceType('m5.4xlarge')], + diskSize: 25, + desiredSize: 2, + maxSize: 3, + nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS } + } + ] + }); + + // const batchTeam: blueprints.BatchEksTeamProps = { + // name: 'batch-a', + // namespace: 'aws-batch', + // envName: 'batch-a-comp-env', + // computeResources: { + // envType: blueprints.BatchEnvType.EC2, + // allocationStrategy: blueprints.BatchAllocationStrategy.BEST, + // priority: 10, + // minvCpus: 0, + // maxvCpus: 128, + // instanceTypes: ["m5", "c4.4xlarge"] + // }, + // jobQueueName: 'team-a-job-queue', + // }; + + blueprints.EksBlueprint.builder() + .resourceProvider(blueprints.GlobalResources.Vpc, vpc) + .resourceProvider('airflow-logging-s3-bucket', apacheAirflowS3Bucket) + // .resourceProvider('airflow-efs-file-system', airflowEfs) + .addOns(...addOns) + .clusterProvider(clusterProvider) + .teams(...teams, + // new blueprints.BatchEksTeam(batchTeam) + ) + // .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) + .build(scope, blueprintID, props); + } +} \ No newline at end of file From 51103c956407f65c978867ba75b7f6a377b40a2d Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 12 Oct 2023 21:29:52 -0500 Subject: [PATCH 07/10] removed local neuron yaml files in place of urls --- .../neuron/k8s-neuron-device-plugin-rbac.ytpl | 58 ----------- .../neuron/k8s-neuron-device-plugin.ytpl | 97 ------------------- 2 files changed, 155 deletions(-) delete mode 100644 lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl delete mode 100644 lib/addons/neuron/k8s-neuron-device-plugin.ytpl diff --git a/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl b/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl deleted file mode 100644 index 28bf10209..000000000 --- a/lib/addons/neuron/k8s-neuron-device-plugin-rbac.ytpl +++ /dev/null @@ -1,58 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: neuron-device-plugin -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - pods - verbs: - - update - - patch - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - update ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: neuron-device-plugin - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: neuron-device-plugin - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: neuron-device-plugin -subjects: -- kind: ServiceAccount - name: neuron-device-plugin - namespace: kube-system diff --git a/lib/addons/neuron/k8s-neuron-device-plugin.ytpl b/lib/addons/neuron/k8s-neuron-device-plugin.ytpl deleted file mode 100644 index 2c909d040..000000000 --- a/lib/addons/neuron/k8s-neuron-device-plugin.ytpl +++ /dev/null @@ -1,97 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: neuron-device-plugin-daemonset - namespace: kube-system -spec: - selector: - matchLabels: - name: neuron-device-plugin-ds - updateStrategy: - type: RollingUpdate - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: neuron-device-plugin-ds - spec: - serviceAccount: neuron-device-plugin - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: aws.amazon.com/neuron - operator: Exists - effect: NoSchedule - # Mark this pod as a critical add-on; when enabled, the critical add-on - # scheduler reserves resources for critical add-on pods so that they can - # be rescheduled after a failure. - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - priorityClassName: "system-node-critical" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "beta.kubernetes.io/instance-type" - operator: In - values: - - inf1.xlarge - - inf1.2xlarge - - inf1.6xlarge - - inf1.24xlarge - - inf2.xlarge - - inf2.4xlarge - - inf2.8xlarge - - inf2.24xlarge - - inf2.48xlarge - - trn1.2xlarge - - trn1.32xlarge - - trn1n.32xlarge - - matchExpressions: - - key: "node.kubernetes.io/instance-type" - operator: In - values: - - inf1.xlarge - - inf1.2xlarge - - inf1.6xlarge - - inf1.24xlarge - - inf2.xlarge - - inf2.4xlarge - - inf2.8xlarge - - inf2.24xlarge - - inf2.48xlarge - - trn1.2xlarge - - trn1.32xlarge - - trn1n.32xlarge - containers: - #Device Plugin containers are available both in us-east and us-west ecr - #repos - - image: public.ecr.aws/neuron/neuron-device-plugin:2.12.5.0 - imagePullPolicy: Always - name: neuron-device-plugin - env: - - name: KUBECONFIG - value: /etc/kubernetes/kubelet.conf - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: infa-map - mountPath: /run - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: infa-map - hostPath: - path: /run - - From b2721118fb764f49a70c476084718f11f1e8a4bb Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 13 Oct 2023 07:59:35 -0500 Subject: [PATCH 08/10] removing local construct test --- examples/young-construct/index.ts | 154 ------------------------------ 1 file changed, 154 deletions(-) delete mode 100644 examples/young-construct/index.ts diff --git a/examples/young-construct/index.ts b/examples/young-construct/index.ts deleted file mode 100644 index 996a49217..000000000 --- a/examples/young-construct/index.ts +++ /dev/null @@ -1,154 +0,0 @@ -import * as cdk from 'aws-cdk-lib'; -import * as ec2 from "aws-cdk-lib/aws-ec2"; -import { KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; -import { AccountRootPrincipal, Role } from 'aws-cdk-lib/aws-iam'; -import { Construct } from "constructs"; -import * as blueprints from '../../lib'; -import { logger, userLog } from '../../lib/utils'; - -export interface BlueprintConstructProps { - /** - * Id - */ - id: string -} - -export default class YoungConstruct { - constructor(scope: Construct, props: cdk.StackProps) { - - blueprints.HelmAddOn.validateHelmVersions = true; - blueprints.HelmAddOn.failOnVersionValidation = false; - logger.settings.minLevel = 3; - userLog.settings.minLevel = 2; - - const vpc = new blueprints.VpcProvider(undefined, { - primaryCidr: "10.2.0.0/16", - secondaryCidr: "100.64.0.0/16", - secondarySubnetCidrs: ["100.64.0.0/24","100.64.1.0/24","100.64.2.0/24"] - }); - // const airflowEfs = new blueprints.CreateEfsFileSystemProvider({ - // name: "airflow-efs-file-system" - // }); - const apacheAirflowS3Bucket = new blueprints.CreateS3BucketProvider({ - id: 'apache-airflow-s3-bucket-id', - s3BucketProps: { removalPolicy: cdk.RemovalPolicy.DESTROY } - }); - - const teams: Array = []; - const addOns: Array = [ - new blueprints.addons.AwsLoadBalancerControllerAddOn(), - new blueprints.addons.CoreDnsAddOn(), - new blueprints.addons.KubeProxyAddOn(), - new blueprints.addons.SSMAgentAddOn(), - // new blueprints.addons.KarpenterAddOn({ - // requirements: [ - // { key: 'node.kubernetes.io/instance-type', op: 'In', vals: ['m5.2xlarge'] }, - // { key: 'topology.kubernetes.io/zone', op: 'NotIn', vals: ['us-west-2c']}, - // { key: 'kubernetes.io/arch', op: 'In', vals: ['amd64','arm64']}, - // { key: 'karpenter.sh/capacity-type', op: 'In', vals: ['spot']}, - // ], - // subnetTags: { - // "Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1", - // }, - // securityGroupTags: { - // "kubernetes.io/cluster/blueprint-construct-dev": "owned", - // }, - // taints: [{ - // key: "workload", - // value: "test", - // effect: "NoSchedule", - // }], - // consolidation: { enabled: true }, - // ttlSecondsUntilExpired: 2592000, - // weight: 20, - // interruptionHandling: true, - // limits: { - // resources: { - // cpu: 20, - // memory: "64Gi", - // } - // }, - // }), - new blueprints.addons.EfsCsiDriverAddOn({replicaCount: 1}), - new blueprints.addons.EbsCsiDriverAddOn(), - // new blueprints.addons.JupyterHubAddOn({ - // efsConfig: { - // pvcName: "efs-persist", - // removalPolicy: cdk.RemovalPolicy.DESTROY, - // capacity: '10Gi', - // }, - // serviceType: blueprints.JupyterHubServiceType.CLUSTERIP, - // notebookStack: 'jupyter/datascience-notebook', - // values: { prePuller: { hook: { enabled: false }}} - // }), - // new blueprints.AwsBatchAddOn(), - new blueprints.AwsForFluentBitAddOn(), - // new blueprints.AirflowAddOn({ - // enableLogging: true, - // s3BucketName: airflowS3bucket.name - // // enableRds: true, - // // dbConfig: { - // // username: "airflow-user", - // // password: "PA$$w0rd123", - // // dbName: "airflow", - // // }, - // // enableEfs: true, - // // efsFileSystemName: airflowEfs.options.name!, - // }), - new blueprints.ApacheAirflowAddOn({ - enableLogging: true, - s3Bucket: 'airflow-logging-s3-bucket', - // enableEfs: true, - // efsFileSystem: 'apache-airflow-efs-provider', - }) - ]; - - const blueprintID = 'young-blueprint-test'; - - const clusterProvider = new blueprints.GenericClusterProvider({ - version: KubernetesVersion.V1_25, - mastersRole: blueprints.getResource(context => { - return new Role(context.scope, 'AdminRole', { assumedBy: new AccountRootPrincipal() }); - }), - managedNodeGroups: [ - { - id: "mng1", - amiType: NodegroupAmiType.AL2_X86_64, - // amiReleaseVersion: "", - instanceTypes: [new ec2.InstanceType('m5.4xlarge')], - diskSize: 25, - desiredSize: 2, - maxSize: 3, - nodeGroupSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS } - } - ] - }); - - // const batchTeam: blueprints.BatchEksTeamProps = { - // name: 'batch-a', - // namespace: 'aws-batch', - // envName: 'batch-a-comp-env', - // computeResources: { - // envType: blueprints.BatchEnvType.EC2, - // allocationStrategy: blueprints.BatchAllocationStrategy.BEST, - // priority: 10, - // minvCpus: 0, - // maxvCpus: 128, - // instanceTypes: ["m5", "c4.4xlarge"] - // }, - // jobQueueName: 'team-a-job-queue', - // }; - - blueprints.EksBlueprint.builder() - .resourceProvider(blueprints.GlobalResources.Vpc, vpc) - .resourceProvider('airflow-logging-s3-bucket', apacheAirflowS3Bucket) - // .resourceProvider('airflow-efs-file-system', airflowEfs) - .addOns(...addOns) - .clusterProvider(clusterProvider) - .teams(...teams, - // new blueprints.BatchEksTeam(batchTeam) - ) - // .enableControlPlaneLogTypes(blueprints.ControlPlaneLogType.API) - .build(scope, blueprintID, props); - } -} \ No newline at end of file From b4f9530455746555c588a22e2fc79821e0d0cbfb Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 13 Oct 2023 10:12:20 -0500 Subject: [PATCH 09/10] removed unneccessary yaml util, added jsdocs on helper functions --- lib/addons/neuron/index.ts | 6 +++--- lib/utils/yaml-utils.ts | 33 ++++++++++++++++++++++++++++----- test/utils/yaml-utils.test.ts | 27 --------------------------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/lib/addons/neuron/index.ts b/lib/addons/neuron/index.ts index dc05f2c29..faf8d3eb7 100644 --- a/lib/addons/neuron/index.ts +++ b/lib/addons/neuron/index.ts @@ -2,7 +2,7 @@ import { Construct } from "constructs"; import { ClusterAddOn, ClusterInfo } from "../../spi"; import { KubectlProvider, ManifestDeployment } from "../helm-addon/kubectl-provider"; -import { loadMultiResourceExternalYaml } from "../../utils/yaml-utils"; +import { loadExternalYaml } from "../../utils/yaml-utils"; const PLUGIN_URL = "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/master/src/k8/k8s-neuron-device-plugin.yml"; const RBAC_URL = "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/master/src/k8/k8s-neuron-device-plugin-rbac.yml"; @@ -12,7 +12,7 @@ export class NeuronPluginAddOn implements ClusterAddOn { const kubectlProvider = new KubectlProvider(clusterInfo); // Read in YAML docs - const rbac = loadMultiResourceExternalYaml(RBAC_URL); + const rbac = loadExternalYaml(RBAC_URL); const rbacManifest: ManifestDeployment = { name: "neuron-rbac-manifest", namespace: "", @@ -20,7 +20,7 @@ export class NeuronPluginAddOn implements ClusterAddOn { values: {} }; - const plugin = loadMultiResourceExternalYaml(PLUGIN_URL); + const plugin = loadExternalYaml(PLUGIN_URL); const pluginManifest: ManifestDeployment = { name: "neuron-plugin-manifest", namespace: "kube-system", diff --git a/lib/utils/yaml-utils.ts b/lib/utils/yaml-utils.ts index e6059b687..dcc5afbb2 100644 --- a/lib/utils/yaml-utils.ts +++ b/lib/utils/yaml-utils.ts @@ -25,6 +25,11 @@ export function applyYamlFromDir(dir: string, cluster: eks.ICluster, namespaceMa }); } +/** + * Reads the YAML document from a local path. + * @param path YAML document path + * @returns YAML document string + */ export function readYamlDocument(path: string): string { try { const doc = fs.readFileSync(path, 'utf8'); @@ -35,20 +40,33 @@ export function readYamlDocument(path: string): string { } } +/** + * Reads the YAML document from a local path and parses them as + * multiple YAML documents separated by `---` as expected in a Kubernetes manifest file + * @param path YAML document path + * @returns a list of parsed YAML documents + */ export function loadMultiResourceYaml(path: string): any { const doc = readYamlDocument(path); return doc.split("---").map((e: any) => loadYaml(e)); } -export function loadMultiResourceExternalYaml(url: string): any { - const doc = loadExternalYaml(url); - return doc; -} - +/** + * Parses the sting document into a single YAML document + * @param document document + * @returns yaml document + */ export function loadYaml(document: string): any { return yaml.load(document); } +/** + * Reads the YAML document from a URL and parses + * multiple YAML documents separated by `---` as expected in a Kubernetes manifest file Note: The file from the URL is + * not validated, so user must ensure the URL contains a valid manifest. + * @param url YAML document URL + * @returns a list of parsed YAML documents + */ export function loadExternalYaml(url: string): any { /* eslint-disable */ const request = require('sync-request'); // moved away from import as it is causing open handles that prevents jest from completion @@ -56,6 +74,11 @@ export function loadExternalYaml(url: string): any { return yaml.loadAll(response.getBody().toString()); } +/** + * Serializes object as a YAML document + * @param document document + * @returns yaml document + */ export function serializeYaml(document: any): string { return yaml.dump(document); } \ No newline at end of file diff --git a/test/utils/yaml-utils.test.ts b/test/utils/yaml-utils.test.ts index 32a51cf23..b1011407e 100644 --- a/test/utils/yaml-utils.test.ts +++ b/test/utils/yaml-utils.test.ts @@ -62,31 +62,4 @@ describe('Unit tests for yaml utils', () => { expect(doc.length).toBe(1); expect(doc[0]).toStrictEqual(part); }); - - test("External YAML Document with multiple resources is read correctly", () => { - const doc = yaml.loadMultiResourceExternalYaml('https://raw.githubusercontent.com/kubernetes/examples/master/guestbook/all-in-one/frontend.yaml'); - const firstPart = { - apiVersion: "v1", - kind: "Service", - metadata: { - name: "frontend", - labels:{ - app: "guestbook", - tier: "frontend" - } - }, - spec: - { - type: "NodePort", - ports: [{port: 80}], - selector: { - app: "guestbook", - tier: "frontend" - } - } - }; - - expect(doc.length).toBe(2); - expect(doc[0]).toStrictEqual(firstPart); - }); }); From c71979a9c3556e66aa7adc42f4cc8d0f14161ebd Mon Sep 17 00:00:00 2001 From: Agus Date: Mon, 12 Feb 2024 10:09:26 +0100 Subject: [PATCH 10/10] changed mkdocks and docs/index.md to point to neuron addon --- docs/addons/index.md | 1 + mkdocs.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/addons/index.md b/docs/addons/index.md index 88699b515..ddb31fdd8 100644 --- a/docs/addons/index.md +++ b/docs/addons/index.md @@ -57,6 +57,7 @@ The framework currently supports the following add-ons. | [`MetricsServerAddOn`](./metrics-server.md) | Adds metrics server (pre-req for HPA and other monitoring tools). | ✅ | ✅ | | [`NewRelicAddOn`](./newrelic.md) | Adds [New Relic](https://newrelic.com/) and [Pixie](https://pixielabs.ai/) observability for Amazon EKS. | ✅ | | [`NginxAddOn`](./nginx.md) | Adds NGINX ingress controller | ✅ | ✅ | | +| [`NeuronAddOn`](./neuron-plugin-addon.md) | Adds Neuron Addon | ✅ | | | [`OpaGatekeeperAddOn`](./opa-gatekeeper.md) | Adds OPA Gatekeeper | ✅ | ✅ | | [`ParalusAddOn`](./paralus.md) | Adds [Paralus](https://paralus.io/) | ✅ | ✅ | | [`PixieAddOn`](./pixie.md) | Adds [Pixie](https://px.dev) to the EKS Cluster. Pixie provides auto-telemetry for requests, metrics, application profiles, and more. | ✅ | diff --git a/mkdocs.yml b/mkdocs.yml index 610143695..5a24e2771 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,6 +68,7 @@ nav: - Metrics Server: 'addons/metrics-server.md' - New Relic: 'addons/newrelic.md' - Nginx: 'addons/nginx.md' + - Neuron: 'addons/neuron-plugin-addon.md' - OPA Gatekeeper: 'addons/opa-gatekeeper.md' - Paralus: 'addons/paralus.md' - Pixie: 'addons/pixie.md'