Karpenter Addon: adding AMI Selector, refactor to use NodeTemplate

docs/addons/karpenter.md

@@ -44,6 +44,9 @@ const karpenterAddonProps = {
     effect: "NoSchedule",
   }],
   amiFamily: "AL2",
+  amiSelector: {
+    "karpenter.sh/discovery/MyClusterName": '*',
+  },
   consolidation: { enabled: true },
   ttlSecondsUntilExpired: 2592000,
   weight: 20,
@@ -81,7 +84,7 @@ blueprints-addon-karpenter-54fd978b89-hclmp   2/2     Running   0          99m
 2. Creates `karpenter` namespace.
 3. Creates Kubernetes Service Account, and associate AWS IAM Role with Karpenter Controller Policy attached using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html).
 4. Deploys Karpenter helm chart in the `karpenter` namespace, configuring cluster name and cluster endpoint on the controller by default.
-5. (Optionally) provisions a default Karpenter Provisioner CRD based on user-provided [spec.requirements](https://karpenter.sh/v0.12.1/provisioner/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family), taints and tags. If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.
+5. (Optionally) provisions a default Karpenter Provisioner and AWSNodeTemplate CRD based on user-provided parameters such as [spec.requirements](https://karpenter.sh/docs/concepts/provisioners/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family),[weight](https://karpenter.sh/docs/concepts/provisioners/#specweight), [Subnet Selector](https://karpenter.sh/docs/concepts/node-templates/#specsubnetselector), and [Security Group Selector](https://karpenter.sh/docs/concepts/node-templates/#specsecuritygroupselector). If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.
 
 **NOTE:**
 1. The default provisioner is created only if both the subnet tags and the security group tags are provided.

examples/blueprint-construct/index.ts

@@ -119,6 +119,9 @@ export default class BlueprintConstruct {
                     value: "test",
                     effect: "NoSchedule",
                 }],
+                amiSelector: {
+                    "karpenter.sh/discovery/MyClusterName": '*',
+                },
                 consolidation: { enabled: true },
                 ttlSecondsUntilExpired: 2592000,
                 weight: 20,

lib/addons/karpenter/index.ts

@@ -19,6 +19,40 @@ import { Cluster } from 'aws-cdk-lib/aws-eks';
  * Configuration options for the add-on
  */
 interface KarpenterAddOnProps extends HelmAddOnUserProps {
+    /**
+     * Taints for the provisioned nodes - Taints may prevent pods from scheduling if they are not tolerated by the pod.
+     */
+    taints?: {
+        key: string,
+        value: string,
+        effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute",
+    }[],
+
+    /**
+     * Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this\
+     * provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for
+     * removing the taint after it has finished initializing the node.
+     */
+    startupTaints?: {
+        key: string,
+        value: string,
+        effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute",
+    },
+
+    /**
+     * Labels applied to all nodes
+     */
+    labels?: {
+        [key: string]: string
+    },
+
+    /**
+     * Annotations applied to all nodes
+     */
+    annotations?: {
+        [key: string]: string
+    }
+
     /**
      * Requirement properties for a Provisioner (Optional) - If not provided, the add-on will
      * deploy a Provisioner with default values.
@@ -29,12 +63,6 @@ interface KarpenterAddOnProps extends HelmAddOnUserProps {
         vals: string[],
     }[]
 
-    taints?: {
-        key: string,
-        value: string,
-        effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute",
-    }[]
-
     /**
      * Tags needed for subnets - Subnet tags and security group tags are required for the provisioner to be created
      */
@@ -54,6 +82,13 @@ interface KarpenterAddOnProps extends HelmAddOnUserProps {
      */
     amiFamily?: "AL2" | "Bottlerocket" | "Ubuntu"
 
+    /**
+     * AMI Selector
+     */
+    amiSelector?: {
+        [key: string]: string
+    }
+
     /**
      * Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those that can't be removed.  
      * Mutually exclusive with the ttlSecondsAfterEmpty parameter.
@@ -117,7 +152,7 @@ const RELEASE = 'blueprints-addon-karpenter';
 const defaultProps: HelmAddOnProps = {
     name: KARPENTER,
     namespace: KARPENTER,
-    version: 'v0.27.3',
+    version: 'v0.28.0',
     chart: KARPENTER,
     release: KARPENTER,
     repository: 'oci://public.ecr.aws/karpenter/karpenter',
@@ -150,7 +185,11 @@ export class KarpenterAddOn extends HelmAddOn {
         const subnetTags = this.options.subnetTags || {};
         const sgTags = this.options.securityGroupTags || {};
         const taints = this.options.taints || [];
+        const startupTaints = this.options.startupTaints || [];
+        const labels = this.options.labels || {};
+        const annotations = this.options.annotations || {};
         const amiFamily = this.options.amiFamily;
+        const amiSelector = this.options.amiSelector;
         const ttlSecondsAfterEmpty = this.options.ttlSecondsAfterEmpty || null;
         const ttlSecondsUntilExpired = this.options.ttlSecondsUntilExpired || null;
         const weight = this.options.weight || null;
@@ -279,21 +318,37 @@ export class KarpenterAddOn extends HelmAddOn {
                 kind: 'Provisioner',
                 metadata: { name: 'default' },
                 spec: {
-                    consolidation: consolidation,
-                    requirements: this.convert(requirements),
+                    providerRef: {
+                        name: "default"
+                    },
                     taints: taints,
+                    startupTaints: startupTaints,
+                    labels: labels,
+                    annotations: annotations,
+                    requirements: this.convert(requirements),
                     limits: limits,
-                    provider: {
-                        amiFamily: amiFamily,
-                        subnetSelector: subnetTags,
-                        securityGroupSelector: sgTags,
-                    },
+                    consolidation: consolidation,
                     ttlSecondsUntilExpired: ttlSecondsUntilExpired,
                     ttlSecondsAfterEmpty: ttlSecondsAfterEmpty,
                     weight: weight,
                 },
             });
             provisioner.node.addDependency(karpenterChart);
+
+            const nodeTemplate = cluster.addManifest('default-node-template', {
+                apiVersion: "karpenter.k8s.aws/v1alpha1",
+                kind: "AWSNodeTemplate",
+                metadata: {
+                    name: "default"
+                },
+                spec: {
+                    amiFamily: amiFamily,
+                    amiSelector: amiSelector,
+                    subnetSelector: subnetTags,
+                    securityGroupSelector: sgTags,
+                },
+            });
+            nodeTemplate.node.addDependency(provisioner);
         }
 
         return Promise.resolve(karpenterChart);