diff --git a/.github/workflows/update-generated.yaml b/.github/workflows/update-generated.yaml index e584663fa2..516e45459e 100644 --- a/.github/workflows/update-generated.yaml +++ b/.github/workflows/update-generated.yaml @@ -2,7 +2,10 @@ name: Update generated files on: workflow_dispatch: {} schedule: - - cron: "0 5 * * Thu" + - cron: "0 5 * * Thu" + +permissions: + id-token: write permissions: id-token: write @@ -15,47 +18,53 @@ jobs: strategy: fail-fast: false matrix: - resource: ["coredns", "aws-node"] + resource: ["coredns", "aws-node", "nvidia-device-plugin"] name: Update ${{ matrix.resource }} and open PR runs-on: ubuntu-latest container: public.ecr.aws/eksctl/eksctl-build:833f4464e865a6398788bf6cbc5447967b8974b7 env: GOPRIVATE: "" steps: - - name: Checkout - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 #v4.1.2 - with: - token: ${{ secrets.EKSCTLBOT_TOKEN }} - fetch-depth: 0 - - name: Configure AWS credentials for coredns update - if: ${{ matrix.resource == 'coredns' }} - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - with: - aws-region: us-west-2 - role-duration-seconds: 900 - role-session-name: eksctl-update-coredns-assets - role-to-assume: ${{ secrets.UPDATE_COREDNS_ROLE_ARN }} - - name: Setup identity as eksctl-bot - uses: ./.github/actions/setup-identity - with: - token: "${{ secrets.EKSCTLBOT_TOKEN }}" - - name: Cache go-build and mod - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: | - ~/.cache/go-build/ - ~/go/pkg/mod/ - key: go-${{ hashFiles('go.sum') }} - restore-keys: | - go- - - name: Update ${{ matrix.resource }} - run: make update-${{ matrix.resource }} - - name: Upsert pull request - uses: peter-evans/create-pull-request@70a41aba780001da0a30141984ae2a0c95d8704e #v6.0.2 - with: - token: ${{ secrets.EKSCTLBOT_TOKEN }} - commit-message: update ${{ matrix.resource }} - committer: eksctl-bot - title: 'Update ${{ matrix.resource }}' - branch: update-${{ matrix.resource }} - labels: area/tech-debt + - name: Checkout + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 #v4.1.2 + with: + token: ${{ secrets.EKSCTLBOT_TOKEN }} + fetch-depth: 0 + - name: Configure AWS credentials for coredns update + if: ${{ matrix.resource == 'coredns' }} + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + aws-region: us-west-2 + role-duration-seconds: 900 + role-session-name: eksctl-update-coredns-assets + role-to-assume: ${{ secrets.UPDATE_COREDNS_ROLE_ARN }} + - name: Setup identity as eksctl-bot + uses: ./.github/actions/setup-identity + with: + token: "${{ secrets.EKSCTLBOT_TOKEN }}" + - name: Cache go-build and mod + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 + with: + path: | + ~/.cache/go-build/ + ~/go/pkg/mod/ + key: go-${{ hashFiles('go.sum') }} + restore-keys: | + go- + - name: Update ${{ matrix.resource }} + run: make update-${{ matrix.resource }} + - name: Upsert pull request + uses: peter-evans/create-pull-request@70a41aba780001da0a30141984ae2a0c95d8704e #v6.0.2 + with: + token: ${{ secrets.EKSCTLBOT_TOKEN }} + commit-message: update ${{ matrix.resource }}${{ env.LATEST_RELEASE_TAG }} + committer: eksctl-bot + title: 'Update ${{ matrix.resource }}${{ env.LATEST_RELEASE_TAG }}' + branch: update-${{ matrix.resource }} + labels: area/tech-debt + body: | + Auto-generated by [eksctl Update Generated Files GitHub workflow][1] + + [1]: https://github.com/eksctl-io/eksctl/blob/main/.github/workflows/update-generated.yaml + + Please manually test before approving and merging. diff --git a/Makefile b/Makefile index be4860e7fe..db9eec9e64 100644 --- a/Makefile +++ b/Makefile @@ -160,6 +160,9 @@ generate-all: generate-always $(conditionally_generated_files) ## Re-generate al check-all-generated-files-up-to-date: generate-all ## Run the generate all command and verify there is no new diff git diff --quiet -- $(conditionally_generated_files) || (git --no-pager diff $(conditionally_generated_files); echo "HINT: to fix this, run 'git commit $(conditionally_generated_files) --message \"Update generated files\"'"; exit 1) +.PHONY: update-nvidia-device-plugin +update-nvidia-device-plugin: ## fetch the latest static manifest + pkg/addons/assets/scripts/update_nvidia_device_plugin.sh .PHONY: update-aws-node update-aws-node: ## Re-download the aws-node manifests from AWS @@ -169,9 +172,6 @@ update-aws-node: ## Re-download the aws-node manifests from AWS update-coredns: ## get latest coredns builds for each available eks version @go run pkg/addons/default/scripts/update_coredns_assets.go -.PHONY: -update-coredns: ## get latest coredns builds for each available eks version - @go run pkg/addons/default/scripts/update_coredns_assets.go deep_copy_helper_input = $(shell $(call godeps_cmd,./pkg/apis/...) | sed 's|$(generated_code_deep_copy_helper)||' ) $(generated_code_deep_copy_helper): $(deep_copy_helper_input) ##Β Generate Kubernetes API helpers diff --git a/docs/release_notes/0.184.0.md b/docs/release_notes/0.184.0.md new file mode 100644 index 0000000000..88ac215fee --- /dev/null +++ b/docs/release_notes/0.184.0.md @@ -0,0 +1,20 @@ +# Release v0.184.0 + +## πŸš€ Features + +- Cluster creation flexibility for default networking addons (#7866) + +## 🎯 Improvements + +- use string in logging instead of wrapping error (#7838) +- Stop using P2 instances which will be retired (#7826) + +## 🧰 Maintenance + +- Fix SDK paginator mocks (#7850) +- Schedule pods on a nodegroup on which no concurrent actions are executed (#7834) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @moreandres. + diff --git a/docs/release_notes/0.185.0.md b/docs/release_notes/0.185.0.md new file mode 100644 index 0000000000..f75add2b83 --- /dev/null +++ b/docs/release_notes/0.185.0.md @@ -0,0 +1,14 @@ +# Release v0.185.0 + +## 🎯 Improvements + +- Avoid creating subnets in disallowed Availability Zone IDs (#7870) +- Add auto-ssm ami resolution for ubuntu (#7851) +- Add/hpc7g node arm support (#6743) +- fix runAsNonRoot (true) efa device plugin bug (#6302) +- fixed iam permissions bug for karpenter (#7778) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @aciba90, @siennathesane and @vsoch. + diff --git a/docs/release_notes/0.186.0.md b/docs/release_notes/0.186.0.md new file mode 100644 index 0000000000..9c878bfe4f --- /dev/null +++ b/docs/release_notes/0.186.0.md @@ -0,0 +1,10 @@ +# Release v0.186.0 + +## πŸš€ Features + +- Allow limiting the number of nodegroups created in parallel (#7884) + +## 🎯 Improvements + +- Retry throttling errors, disable retry rate-limiting (#7878) + diff --git a/docs/release_notes/0.187.0.md b/docs/release_notes/0.187.0.md new file mode 100644 index 0000000000..5417862471 --- /dev/null +++ b/docs/release_notes/0.187.0.md @@ -0,0 +1,5 @@ +# Release v0.187.0 + +## πŸ› Bug Fixes + +- Restrict `VPC.SecurityGroup` egress rules validations to self-managed nodes (#7883) diff --git a/docs/release_notes/0.188.0.md b/docs/release_notes/0.188.0.md new file mode 100644 index 0000000000..50bb6043eb --- /dev/null +++ b/docs/release_notes/0.188.0.md @@ -0,0 +1,14 @@ +# Release v0.188.0 + +## πŸš€ Features + +- Add support for Kuala Lumpur region (ap-southeast-5) (#7910) + +## 🎯 Improvements + +- Add GH workflow for automatically updating nvidia device plugin static manifest (#7898) + +## 🧰 Maintenance + +- Update nvidia-device-plugin to v0.16.0 (#7900) + diff --git a/docs/release_notes/0.189.0.md b/docs/release_notes/0.189.0.md new file mode 100644 index 0000000000..3e86418441 --- /dev/null +++ b/docs/release_notes/0.189.0.md @@ -0,0 +1,12 @@ +# Release v0.189.0 + +## πŸ› Bug Fixes + +- Skip creating OIDC manager for Outposts clusters (#7934) +- Fixes segfault when VPC CNI is disabled (#7927) +- Fix SSM unit tests (#7935) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @EmmEff. + diff --git a/docs/release_notes/0.190.0.md b/docs/release_notes/0.190.0.md new file mode 100644 index 0000000000..136aa692c7 --- /dev/null +++ b/docs/release_notes/0.190.0.md @@ -0,0 +1,6 @@ +# Release v0.190.0 + +## 🧰 Maintenance + +- Bump github.com/docker/docker from 26.1.4+incompatible to 26.1.5+incompatible (#7936) +- Bump jinja2 from 3.1.3 to 3.1.4 in /userdocs (#7748) diff --git a/docs/release_notes/0.191.0.md b/docs/release_notes/0.191.0.md new file mode 100644 index 0000000000..fac29caf8d --- /dev/null +++ b/docs/release_notes/0.191.0.md @@ -0,0 +1,5 @@ +# Release v0.191.0 + +## πŸš€ Features + +- Add support for EKS 1.31 (#7973) diff --git a/docs/release_notes/0.192.0.md b/docs/release_notes/0.192.0.md new file mode 100644 index 0000000000..aaa3d8d286 --- /dev/null +++ b/docs/release_notes/0.192.0.md @@ -0,0 +1,18 @@ +# Release v0.192.0 + +## πŸš€ Features + +- Add support for EKS accelerated AMIs based on AL2023 (#7996) + +## 🎯 Improvements + +- cleanup efa installer archive before install (#6870) + +## πŸ› Bug Fixes + +- Disallow `overrideBootstrapCommand` and `preBootstrapCommands` for MNG AL2023 (#7990) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @vsoch. + diff --git a/docs/release_notes/0.193.0.md b/docs/release_notes/0.193.0.md new file mode 100644 index 0000000000..c3f5a2eb82 --- /dev/null +++ b/docs/release_notes/0.193.0.md @@ -0,0 +1,14 @@ +# Release v0.193.0 + +## πŸš€ Features + +- Add support for M8g instance types (#8001) + +## πŸ“ Documentation + +- Documentation: correct version drift limits in cluster-upgrade.md (#7994) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @larvacea. + diff --git a/docs/release_notes/0.194.0.md b/docs/release_notes/0.194.0.md new file mode 100644 index 0000000000..ef8ab41317 --- /dev/null +++ b/docs/release_notes/0.194.0.md @@ -0,0 +1,14 @@ +# Release v0.194.0 + +## πŸš€ Features + +- Support EKS zonal shift config (#8005) + +## 🎯 Improvements + +- Fix missing ELB listener attribute actions required for AWS Load Balancer Controller v2.9.0 (#7988) + +## Acknowledgments + +The eksctl maintainers would like to sincerely thank @jonathanfoster. + diff --git a/examples/41-zonal-shift.yaml b/examples/41-zonal-shift.yaml new file mode 100644 index 0000000000..8536560fb4 --- /dev/null +++ b/examples/41-zonal-shift.yaml @@ -0,0 +1,11 @@ +# An example ClusterConfig that uses EKS Zonal Shift. + +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: highly-available-cluster + region: us-west-2 + +zonalShiftConfig: + enabled: true diff --git a/go.mod b/go.mod index 9d24056aba..6bff582330 100644 --- a/go.mod +++ b/go.mod @@ -5,20 +5,22 @@ module github.com/weaveworks/eksctl go 1.21 +toolchain go1.21.5 + require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/aws/amazon-ec2-instance-selector/v2 v2.4.2-0.20230601180523-74e721cb8c1e github.com/aws/aws-sdk-go v1.51.16 - github.com/aws/aws-sdk-go-v2 v1.27.1 + github.com/aws/aws-sdk-go-v2 v1.32.2 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/credentials v1.17.11 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.5 - github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.52.1 github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.2 github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1 github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider v1.36.3 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.156.0 - github.com/aws/aws-sdk-go-v2/service/eks v1.43.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.166.0 + github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.4 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 @@ -26,7 +28,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/outposts v1.38.0 github.com/aws/aws-sdk-go-v2/service/ssm v1.49.5 github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 - github.com/aws/smithy-go v1.20.2 + github.com/aws/smithy-go v1.22.0 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240508073157-fbfa1bc129f5 github.com/benjamintf1/unmarshalledmatchers v1.0.0 github.com/blang/semver v3.5.1+incompatible @@ -60,14 +62,14 @@ require ( github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/spotinst/spotinst-sdk-go v1.171.0 + github.com/spotinst/spotinst-sdk-go v1.372.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 github.com/tidwall/sjson v1.2.5 github.com/tj/assert v0.0.3 github.com/vburenin/ifacemaker v1.2.1 github.com/vektra/mockery/v2 v2.38.0 - github.com/weaveworks/goformation/v4 v4.10.2-0.20231113122203-bf1ae633f95c + github.com/weaveworks/goformation/v4 v4.10.2-0.20241022124128-4be25b69f5e0 github.com/weaveworks/schemer v0.0.0-20230525114451-47139fe25848 github.com/xgfone/netaddr v0.5.1 golang.org/x/crypto v0.22.0 @@ -126,11 +128,11 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 // indirect github.com/aws/aws-sdk-go-v2/service/pricing v1.17.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect @@ -165,13 +167,13 @@ require ( github.com/daixiang0/gci v0.12.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/docker/cli v24.0.6+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.9+incompatible // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/cli v25.0.1+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker v26.1.5+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.0 // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/ettle/strcase v0.2.0 // indirect @@ -298,7 +300,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/moricho/tparallel v0.3.1 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect @@ -312,7 +313,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opencontainers/image-spec v1.1.0-rc6 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect @@ -422,7 +423,7 @@ require ( k8s.io/kubectl v0.29.0 // indirect mvdan.cc/gofumpt v0.6.0 // indirect mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 // indirect - oras.land/oras-go v1.2.4 // indirect + oras.land/oras-go v1.2.5 // indirect sigs.k8s.io/controller-runtime v0.17.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect diff --git a/go.sum b/go.sum index 109c746402..b2c017e847 100644 --- a/go.sum +++ b/go.sum @@ -716,8 +716,8 @@ github.com/aws/amazon-ec2-instance-selector/v2 v2.4.2-0.20230601180523-74e721cb8 github.com/aws/aws-sdk-go v1.51.16 h1:vnWKK8KjbftEkuPX8bRj3WHsLy1uhotn0eXptpvrxJI= github.com/aws/aws-sdk-go v1.51.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.16.15/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2 v1.27.1 h1:xypCL2owhog46iFxBKKpBcw+bPTX/RJzwNj8uSilENw= -github.com/aws/aws-sdk-go-v2 v1.27.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= @@ -727,27 +727,27 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22/go.mod h1:/vNv5Al0bpiF8YdX2Ov6Xy05VTiXsql94yUqJMYaj0w= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8 h1:RnLB7p6aaFMRfyQkD6ckxR7myCC9SABIqSz4czYUUbU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8/go.mod h1:XH7dQJd+56wEbP1I4e4Duo+QhSMxNArE8VP7NuUOTeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16/go.mod h1:62dsXI0BqTIGomDl8Hpm33dv0OntGaVblri3ZRParVQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8 h1:jzApk2f58L9yW9q1GEab3BMMFWUkkiZhyrRUtbwUbKU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8/go.mod h1:WqO+FftfO3tGePUtQxPXM6iODVfqMwsVMgTbG/ZXIdQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.5 h1:vhdJymxlWS2qftzLiuCjSswjXBRLGfzo/BEE9LDveBA= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.5/go.mod h1:ZErgk/bPaaZIpj+lUWGlwI1A0UFhSIscgnCPzTLnb2s= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.52.1 h1:Ts+mCjOtt8o2k2vnWnX/0sE0eSmEVWBvfJkNrNMQlAo= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.52.1/go.mod h1:IrWhabzdTEc651GAq7rgst/SYcEqqcD7Avr82m28AAU= github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.2 h1:svl3DNKWpcLOlz+bFzmOxGp8gcbvSZ6m2t44Zzaet9U= github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.2/go.mod h1:gAJs+mKIoK4JTQD1KMZtHgyBRZ8S6Oy5+qjJzoDAvbE= github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1 h1:suWu59CRsDNhw2YXPpa6drYEetIUUIMUhkzHmucbCf8= github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1/go.mod h1:tZiRxrv5yBRgZ9Z4OOOxwscAZRFk5DgYhEcjX1QpvgI= github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider v1.36.3 h1:JNWpkjImTP2e308bv7ihfwgOawf640BY/pyZWrBb9rw= github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider v1.36.3/go.mod h1:TiLZ2/+WAEyG2PnuAYj/un46UJ7qBf5BWWTAKgaHP8I= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.156.0 h1:TFK9GeUINErClL2+A+GLYhjiChVdaXCgIUiCsS/UQrE= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.156.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8= -github.com/aws/aws-sdk-go-v2/service/eks v1.43.0 h1:TRgA51vdnrXiZpCab7pQT0bF52rX5idH0/fzrIVnQS0= -github.com/aws/aws-sdk-go-v2/service/eks v1.43.0/go.mod h1:875ZmajQCZ9N7HeR1DE25nTSaalkqGYzQa+BxLattlQ= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.166.0 h1:FDZVMxzXB13cRmHs3t3tH9gme8GhvmjsQXeXFI37OHU= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.166.0/go.mod h1:Wv7N3iFOKVsZNIaw9MOBUmwCkX6VMmQQRFhMrHtNGno= +github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 h1:BYyB+byjQ7oyupe3v+YjTp1yfmfNEwChYA2naCc85xI= +github.com/aws/aws-sdk-go-v2/service/eks v1.51.0/go.mod h1:oaPCqTzAe8C5RQZJGRD4RENcV7A4n99uGxbD4rULbNg= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.4 h1:V5YvSMQwZklktzYeOOhYdptx7rP650XP3RnxwNu1UEQ= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.4/go.mod h1:aYygRYqRxmLGrxRxAisgNarwo4x8bcJG14rh4r57VqE= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 h1:/x2u/TOx+n17U+gz98TOw1HKJom0EOqrhL4SjrHr0cQ= @@ -756,8 +756,8 @@ github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 h1:ZNlfPdw849gBo/lvLFbEEvpTJMij github.com/aws/aws-sdk-go-v2/service/iam v1.32.0/go.mod h1:aXWImQV0uTW35LM0A/T4wEg6R1/ReXUu4SM6/lUHYK0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 h1:zSDPny/pVnkqABXYRicYuPf9z2bTqfH13HT3v6UheIk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14/go.mod h1:3TTcI5JSzda1nw/pkVC9dhgLre0SNBFj2lYS4GctXKI= github.com/aws/aws-sdk-go-v2/service/kms v1.27.5 h1:7lKTr8zJ2nVaVgyII+7hUayTi7xWedMuANiNVXiD2S8= github.com/aws/aws-sdk-go-v2/service/kms v1.27.5/go.mod h1:D9FVDkZjkZnnFHymJ3fPVz0zOUlNSd0xcIIVmmrAac8= github.com/aws/aws-sdk-go-v2/service/outposts v1.38.0 h1:e4uIyH2aMFUtUaHjO/NCNBkXdxBBJj3OnSM5pMo5i0s= @@ -773,8 +773,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEj github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= -github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240508073157-fbfa1bc129f5 h1:F80UWAvCDH3PgWIkMhwhKN7FRlkn9MhI+nBHFq739ZM= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240508073157-fbfa1bc129f5/go.mod h1:wLKtvVfT0IdSJ3Pf6QoeLN+UTUeU28CmSAnoja6/l5s= github.com/awslabs/goformation/v4 v4.19.5 h1:Y+Tzh01tWg8gf//AgGKUamaja7Wx9NPiJf1FpZu4/iU= @@ -907,24 +907,24 @@ github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42 github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlespiau/kube-test-harness v0.0.0-20200915102055-a03579200ae8 h1:pxDCsB4pEs/4FG8pEnNHG7rzr8RvDEdLyDGL653gnB0= github.com/dlespiau/kube-test-harness v0.0.0-20200915102055-a03579200ae8/go.mod h1:DPS/2w0SxCgLfTwNw+/806eccMQ1WjgHb1B70w75wSk= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= -github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= +github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -1489,8 +1489,6 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= @@ -1571,8 +1569,8 @@ github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= +github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= @@ -1742,8 +1740,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= -github.com/spotinst/spotinst-sdk-go v1.171.0 h1:ZihMPEjkpIkSpawWLJt9RtCRY4mOQMGlfrkVmA03000= -github.com/spotinst/spotinst-sdk-go v1.171.0/go.mod h1:Ku9c4p+kRWnQqmXkzGcTMHLcQKgLHrQZISxeKY7mPqE= +github.com/spotinst/spotinst-sdk-go v1.372.0 h1:B4/+HK3D2Fe0821DOmw5RO4Lrzo2gi7oa6QWWjr5/7A= +github.com/spotinst/spotinst-sdk-go v1.372.0/go.mod h1:Tn4/eb0SFY6IXmxz71CClujvbD/PuT+EO6Ta8v6AML4= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= @@ -1815,8 +1813,8 @@ github.com/vektra/mockery/v2 v2.38.0 h1:I0LBuUzZHqAU4d1DknW0DTFBPO6n8TaD38WL2KJf github.com/vektra/mockery/v2 v2.38.0/go.mod h1:diB13hxXG6QrTR0ol2Rk8s2dRMftzvExSvPDKr+IYKk= github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 h1:txplJASvd6b/hrE0s/Ixfpp2cuwH9IO9oZBAN9iYa4A= github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2/go.mod h1:DGCIhurYgnLz8J9ga1fMV/fbLDyUvTyrWXVWUIyJon4= -github.com/weaveworks/goformation/v4 v4.10.2-0.20231113122203-bf1ae633f95c h1:iejfxgm8iQ6Jr3yT7Javgk40drlL6w9B6+zgACs0fMw= -github.com/weaveworks/goformation/v4 v4.10.2-0.20231113122203-bf1ae633f95c/go.mod h1:3c2tyJmoge5qTS4PXS0niVJxR0YzroIBsts3dQI3EdI= +github.com/weaveworks/goformation/v4 v4.10.2-0.20241022124128-4be25b69f5e0 h1:wLOxc4ZnbvJdc7d7b+u4LqgPO5DJrumSXi7Ezo/lYvI= +github.com/weaveworks/goformation/v4 v4.10.2-0.20241022124128-4be25b69f5e0/go.mod h1:3c2tyJmoge5qTS4PXS0niVJxR0YzroIBsts3dQI3EdI= github.com/weaveworks/schemer v0.0.0-20230525114451-47139fe25848 h1:I7S+IHZIU49skVgTNArf9bIdy07mCn1Z0zv1r07ROws= github.com/weaveworks/schemer v0.0.0-20230525114451-47139fe25848/go.mod h1:y8Luzq6JDsYVoIV0QAlnvIiq8bSaap0myMjWKyzVFTY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -2782,8 +2780,8 @@ mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo= mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 h1:zCr3iRRgdk5eIikZNDphGcM6KGVTx3Yu+/Uu9Es254w= mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14/go.mod h1:ZzZjEpJDOmx8TdVU6umamY3Xy0UAQUI2DHbf05USVbI= -oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= -oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= +oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= +oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/integration/data/crud-podinfo.yaml b/integration/data/crud-podinfo.yaml new file mode 100644 index 0000000000..8cbbbea980 --- /dev/null +++ b/integration/data/crud-podinfo.yaml @@ -0,0 +1,53 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + labels: + app: podinfo +spec: + replicas: 2 + selector: + matchLabels: + app: podinfo + template: + metadata: + labels: + app: podinfo + annotations: + prometheus.io/scrape: 'true' + spec: + nodeSelector: + used-for: test-pods + containers: + - name: podinfod + image: stefanprodan/podinfo:1.5.1@sha256:702633d438950f3675d0763a4ca6cfcf21a4d065cd7f470446c67607b1a26750 + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + runAsUser: 1000 + command: + - ./podinfo + - --port=8080 + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8080 + initialDelaySeconds: 1 + periodSeconds: 5 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 1 + periodSeconds: 10 + failureThreshold: 2 + resources: + requests: + memory: "32Mi" + cpu: "10m" diff --git a/integration/data/iamserviceaccount-checker.yaml b/integration/data/iamserviceaccount-checker.yaml index ee79ee7505..5f9226d695 100644 --- a/integration/data/iamserviceaccount-checker.yaml +++ b/integration/data/iamserviceaccount-checker.yaml @@ -18,6 +18,8 @@ spec: serviceAccountName: s3-reader # use a shared volume volumes: [{name: aws-credentials, emptyDir: {}}] + nodeSelector: + used-for: test-pods initContainers: - name: assume-role image: atlassian/pipelines-awscli:1.16.185 diff --git a/integration/data/test-dns.yaml b/integration/data/test-dns.yaml index 2178f8e287..abfbb7bf45 100644 --- a/integration/data/test-dns.yaml +++ b/integration/data/test-dns.yaml @@ -10,6 +10,8 @@ spec: metadata: labels: {test: dns} spec: + nodeSelector: + used-for: test-pods containers: - image: tutum/dnsutils@sha256:d2244ad47219529f1003bd1513f5c99e71655353a3a63624ea9cb19f8393d5fe name: dns-test-cluster diff --git a/integration/data/test-http.yaml b/integration/data/test-http.yaml index 1a2eaf944f..030d938c5b 100644 --- a/integration/data/test-http.yaml +++ b/integration/data/test-http.yaml @@ -10,6 +10,8 @@ spec: metadata: labels: {test: http} spec: + nodeSelector: + used-for: test-pods containers: - image: curlimages/curl@sha256:fa32ef426092b88ee0b569d6f81ab0203ee527692a94ec2e6ceb2fd0b6b2755c name: https-test-eksctl diff --git a/integration/tests/addons/addons_test.go b/integration/tests/addons/addons_test.go index 8dfcf7d6ed..b2be701bfb 100644 --- a/integration/tests/addons/addons_test.go +++ b/integration/tests/addons/addons_test.go @@ -107,31 +107,8 @@ var _ = Describe("(Integration) [EKS Addons test]", func() { return cmd }, "5m", "30s").Should(RunSuccessfullyWithOutputStringLines(ContainElement(ContainSubstring("ACTIVE")))) - By("successfully creating the kube-proxy addon") - cmd := params.EksctlCreateCmd. - WithArgs( - "addon", - "--name", "kube-proxy", - "--cluster", clusterName, - "--force", - "--wait", - "--verbose", "2", - ) - Expect(cmd).To(RunSuccessfully()) - - Eventually(func() runner.Cmd { - cmd := params.EksctlGetCmd. - WithArgs( - "addon", - "--name", "kube-proxy", - "--cluster", clusterName, - "--verbose", "2", - ) - return cmd - }, "5m", "30s").Should(RunSuccessfullyWithOutputStringLines(ContainElement(ContainSubstring("ACTIVE")))) - By("Deleting the kube-proxy addon") - cmd = params.EksctlDeleteCmd. + cmd := params.EksctlDeleteCmd. WithArgs( "addon", "--name", "kube-proxy", @@ -155,12 +132,45 @@ var _ = Describe("(Integration) [EKS Addons test]", func() { }) It("should have full control over configMap when creating addons", func() { - var ( - clusterConfig *api.ClusterConfig - configMap *corev1.ConfigMap - ) + clusterConfig := getInitialClusterConfig() + clusterConfig.Addons = []*api.Addon{ + { + Name: "coredns", + Version: "latest", + }, + } + cmd := params.EksctlCreateCmd. + WithArgs( + "addon", + "--config-file", "-", + ). + WithoutArg("--region", params.Region). + WithStdin(clusterutils.Reader(clusterConfig)) + Expect(cmd).To(RunSuccessfully()) - configMap = getConfigMap(rawClient.ClientSet(), "coredns") + By("deleting coredns but preserving its resources") + cmd = params.EksctlDeleteCmd. + WithArgs( + "addon", + "--cluster", clusterConfig.Metadata.Name, + "--name", "coredns", + "--verbose", "4", + "--preserve", + "--region", params.Region, + ) + Expect(cmd).To(RunSuccessfully()) + + Eventually(func() runner.Cmd { + return params.EksctlGetCmd. + WithArgs( + "addon", + "--name", "coredns", + "--cluster", clusterName, + "--verbose", "4", + ) + }, "5m", "30s").ShouldNot(RunSuccessfully()) + + configMap := getConfigMap(rawClient.ClientSet(), "coredns") oldCacheValue := getCacheValue(configMap) newCacheValue := addToString(oldCacheValue, 5) updateCacheValue(configMap, oldCacheValue, newCacheValue) @@ -178,14 +188,14 @@ var _ = Describe("(Integration) [EKS Addons test]", func() { data, err := json.Marshal(clusterConfig) Expect(err).NotTo(HaveOccurred()) - cmd := params.EksctlCreateCmd. + cmd = params.EksctlCreateCmd. WithArgs( "addon", "--config-file", "-", ). WithoutArg("--region", params.Region). WithStdin(bytes.NewReader(data)) - Expect(cmd).ShouldNot(RunSuccessfully()) + Expect(cmd).NotTo(RunSuccessfully()) Eventually(func() runner.Cmd { cmd := params.EksctlGetCmd. @@ -866,7 +876,11 @@ func getInitialClusterConfig() *api.ClusterConfig { Name: "vpc-cni", AttachPolicyARNs: []string{"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"}, }, + { + Name: "kube-proxy", + }, } + clusterConfig.AddonsConfig.DisableDefaultAddons = true ng := &api.ManagedNodeGroup{ NodeGroupBase: &api.NodeGroupBase{ diff --git a/integration/tests/bare_cluster/bare_cluster_test.go b/integration/tests/bare_cluster/bare_cluster_test.go new file mode 100644 index 0000000000..439f0ce9ae --- /dev/null +++ b/integration/tests/bare_cluster/bare_cluster_test.go @@ -0,0 +1,85 @@ +//go:build integration +// +build integration + +//revive:disable Not changing package name +package bare_cluster + +import ( + "context" + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/weaveworks/eksctl/integration/runner" + "github.com/weaveworks/eksctl/integration/tests" + clusterutils "github.com/weaveworks/eksctl/integration/utilities/cluster" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/testutils" +) + +var params *tests.Params + +func init() { + testing.Init() + params = tests.NewParams("bare-cluster") +} + +func TestBareCluster(t *testing.T) { + testutils.RegisterAndRun(t) +} + +var _ = Describe("Bare Clusters", Ordered, func() { + var clusterConfig *api.ClusterConfig + + BeforeAll(func() { + By("creating a cluster with only VPC CNI and no other default networking addons") + clusterConfig = api.NewClusterConfig() + clusterConfig.Metadata.Name = params.ClusterName + clusterConfig.Metadata.Region = params.Region + clusterConfig.AddonsConfig.DisableDefaultAddons = true + clusterConfig.Addons = []*api.Addon{ + { + Name: "vpc-cni", + UseDefaultPodIdentityAssociations: true, + }, + { + Name: "eks-pod-identity-agent", + }, + } + cmd := params.EksctlCreateCmd. + WithArgs( + "cluster", + "--config-file=-", + "--verbose", "4", + "--kubeconfig="+params.KubeconfigPath, + ). + WithoutArg("--region", params.Region). + WithStdin(clusterutils.Reader(clusterConfig)) + + Expect(cmd).To(RunSuccessfully()) + }) + + It("should have only VPC CNI installed", func() { + config, err := clientcmd.BuildConfigFromFlags("", params.KubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + clientset, err := kubernetes.NewForConfig(config) + Expect(err).NotTo(HaveOccurred()) + _, err = clientset.AppsV1().Deployments(metav1.NamespaceSystem).Get(context.Background(), "coredns", metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "expected coredns to not exist") + daemonSets := clientset.AppsV1().DaemonSets(metav1.NamespaceSystem) + _, err = daemonSets.Get(context.Background(), "kube-proxy", metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "expected kube-proxy to not exist") + _, err = daemonSets.Get(context.Background(), "aws-node", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "expected aws-node to exist") + }) +}) + +var _ = AfterSuite(func() { + params.DeleteClusters() +}) diff --git a/integration/tests/crud/creategetdelete_test.go b/integration/tests/crud/creategetdelete_test.go index 4b8d6f92de..fb696d1fa9 100644 --- a/integration/tests/crud/creategetdelete_test.go +++ b/integration/tests/crud/creategetdelete_test.go @@ -66,6 +66,7 @@ func TestCRUD(t *testing.T) { } const ( + deployNg = "ng-deploy" deleteNg = "ng-delete" taintsNg1 = "ng-taints-1" taintsNg2 = "ng-taints-2" @@ -131,6 +132,17 @@ var _ = SynchronizedBeforeSuite(func() []byte { }, } cfg.ManagedNodeGroups = []*api.ManagedNodeGroup{ + { + NodeGroupBase: &api.NodeGroupBase{ + Name: deployNg, + ScalingConfig: &api.ScalingConfig{ + DesiredCapacity: aws.Int(5), + }, + Labels: map[string]string{ + "used-for": "test-pods", + }, + }, + }, { NodeGroupBase: &api.NodeGroupBase{ Name: drainMng, @@ -214,7 +226,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { Expect(session.ExitCode()).To(BeZero()) var stacks []*cfntypes.Stack Expect(yaml.Unmarshal(session.Out.Contents(), &stacks)).To(Succeed()) - Expect(stacks).To(HaveLen(6)) + Expect(stacks).To(HaveLen(7)) var ( names, descriptions []string @@ -227,6 +239,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { Expect(names).To(ContainElements( ContainSubstring(params.ClusterName+"-cluster"), + ContainSubstring(ngPrefix+deployNg), ContainSubstring(ngPrefix+deleteNg), ContainSubstring(ngPrefix+scaleSingleNg), ContainSubstring(ngPrefix+scaleMultipleNg), @@ -261,7 +274,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { }) It("should deploy podinfo service to the cluster and access it via proxy", func() { - d := test.CreateDeploymentFromFile(test.Namespace, "../../data/podinfo.yaml") + d := test.CreateDeploymentFromFile(test.Namespace, "../../data/crud-podinfo.yaml") test.WaitForDeploymentReady(d, commonTimeout) pods := test.ListPodsFromDeployment(d) @@ -520,7 +533,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { "1.1.1.1/32,2.2.2.0/24", "--approve", )).To(RunSuccessfully()) - Expect(k8sAPICall()).Should(HaveOccurred()) + Eventually(k8sAPICall, "5m", "20s").Should(HaveOccurred()) }) It("should re-enable public access", func() { @@ -898,7 +911,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { "--timeout=45m", "--cluster", params.ClusterName, "--nodes", "1", - "--instance-types", "p2.xlarge,p3.2xlarge,p3.8xlarge,g3s.xlarge,g4ad.xlarge,g4ad.2xlarge", + "--instance-types", "p3.2xlarge,p3.8xlarge,g3s.xlarge,g4ad.xlarge,g4ad.2xlarge", "--node-private-networking", "--node-zones", "us-west-2b,us-west-2c", GPUMng, @@ -970,7 +983,6 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { "--timeout", time.Hour.String(), "--cluster", params.ClusterName, "--nodes", "1", - "--node-type", "p2.xlarge", "--subnet-ids", extraSubnetID, newSubnetCLIMng, )).To(RunSuccessfully()) @@ -1197,6 +1209,7 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { "-o", "json", "--cluster", params.ClusterName, )).To(RunSuccessfullyWithOutputString(BeNodeGroupsWithNamesWhich( + ContainElement(deployNg), ContainElement(taintsNg1), ContainElement(taintsNg2), ContainElement(scaleSingleNg), diff --git a/integration/tests/update/update_cluster_test.go b/integration/tests/update/update_cluster_test.go index 336211c3aa..7f91146d91 100644 --- a/integration/tests/update/update_cluster_test.go +++ b/integration/tests/update/update_cluster_test.go @@ -14,9 +14,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/eks/types" "github.com/aws/aws-sdk-go/aws" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -39,8 +41,9 @@ const ( ) var ( - defaultCluster string - params *tests.Params + defaultCluster string + params *tests.Params + clusterProvider *eks.ClusterProvider ) func init() { @@ -137,6 +140,10 @@ var _ = BeforeSuite(func() { WithoutArg("--region", params.Region). WithStdin(clusterutils.Reader(clusterConfig)) Expect(cmd).To(RunSuccessfully()) + + var err error + clusterProvider, err = newClusterProvider(context.Background()) + Expect(err).NotTo(HaveOccurred()) }) var _ = Describe("(Integration) Upgrading cluster", func() { @@ -175,17 +182,47 @@ var _ = Describe("(Integration) Upgrading cluster", func() { }) }) + Context("default networking addons", func() { + defaultNetworkingAddons := []string{"vpc-cni", "kube-proxy", "coredns"} + + It("should suggest using `eksctl update addon` for updating default addons", func() { + assertAddonError := func(updateAddonName, addonName string) { + cmd := params.EksctlUtilsCmd.WithArgs( + fmt.Sprintf("update-%s", updateAddonName), + "--cluster", params.ClusterName, + "--verbose", "4", + "--approve", + ) + session := cmd.Run() + ExpectWithOffset(1, session.ExitCode()).NotTo(BeZero()) + ExpectWithOffset(1, string(session.Err.Contents())).To(ContainSubstring("Error: addon %s is installed as a managed EKS addon; "+ + "to update it, use `eksctl update addon` instead", addonName)) + } + assertAddonError("aws-node", "vpc-cni") + for _, addonName := range defaultNetworkingAddons { + updateAddonName := addonName + if addonName == "vpc-cni" { + updateAddonName = "aws-node" + } + assertAddonError(updateAddonName, addonName) + } + }) + }) + Context("addons", func() { It("should upgrade kube-proxy", func() { - cmd := params.EksctlUtilsCmd.WithArgs( - "update-kube-proxy", - "--cluster", params.ClusterName, - "--verbose", "4", - "--approve", - ) + cmd := params.EksctlUpdateCmd. + WithArgs( + "addon", + "--name", "kube-proxy", + "--cluster", params.ClusterName, + "--version", "latest", + "--wait", + "--verbose", "4", + ) Expect(cmd).To(RunSuccessfully()) - rawClient := getRawClient(context.Background()) + rawClient := getRawClient(context.Background(), clusterProvider) Eventually(func() string { daemonSet, err := rawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.TODO(), "kube-proxy", metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -200,7 +237,7 @@ var _ = Describe("(Integration) Upgrading cluster", func() { }) It("should upgrade aws-node", func() { - rawClient := getRawClient(context.Background()) + rawClient := getRawClient(context.Background(), clusterProvider) getAWSNodeVersion := func() string { awsNode, err := rawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.TODO(), "aws-node", metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -210,24 +247,29 @@ var _ = Describe("(Integration) Upgrading cluster", func() { } preUpdateAWSNodeVersion := getAWSNodeVersion() - cmd := params.EksctlUtilsCmd.WithArgs( - "update-aws-node", - "--cluster", params.ClusterName, - "--verbose", "4", - "--approve", - ) + cmd := params.EksctlUpdateCmd. + WithArgs( + "addon", + "--name", "vpc-cni", + "--cluster", params.ClusterName, + "--version", "latest", + "--wait", + "--verbose", "4", + ) Expect(cmd).To(RunSuccessfully()) - Eventually(getAWSNodeVersion, k8sUpdatePollTimeout, k8sUpdatePollInterval).ShouldNot(Equal(preUpdateAWSNodeVersion)) }) It("should upgrade coredns", func() { - cmd := params.EksctlUtilsCmd.WithArgs( - "update-coredns", - "--cluster", params.ClusterName, - "--verbose", "4", - "--approve", - ) + cmd := params.EksctlUpdateCmd. + WithArgs( + "addon", + "--name", "coredns", + "--cluster", params.ClusterName, + "--version", "latest", + "--wait", + "--verbose", "4", + ) Expect(cmd).To(RunSuccessfully()) }) @@ -285,19 +327,36 @@ var _ = AfterSuite(func() { os.RemoveAll(params.TestDirectory) }) -func getRawClient(ctx context.Context) *kubewrapper.RawClient { +func newClusterProvider(ctx context.Context) (*eks.ClusterProvider, error) { cfg := &api.ClusterConfig{ Metadata: &api.ClusterMeta{ Name: params.ClusterName, Region: params.Region, }, } - ctl, err := eks.New(context.TODO(), &api.ProviderConfig{Region: params.Region}, cfg) - Expect(err).NotTo(HaveOccurred()) + ctl, err := eks.New(ctx, &api.ProviderConfig{Region: params.Region}, cfg) + if err != nil { + return nil, err + } + if err := ctl.RefreshClusterStatus(ctx, cfg); err != nil { + return nil, err + } + return ctl, nil +} + +func defaultClusterConfig() *api.ClusterConfig { + return &api.ClusterConfig{ + Metadata: &api.ClusterMeta{ + Name: params.ClusterName, + Region: params.Region, + }, + } +} - err = ctl.RefreshClusterStatus(ctx, cfg) - Expect(err).ShouldNot(HaveOccurred()) - rawClient, err := ctl.NewRawClient(cfg) +func getRawClient(ctx context.Context, ctl *eks.ClusterProvider) *kubewrapper.RawClient { + clusterConfig := defaultClusterConfig() + Expect(ctl.RefreshClusterStatus(ctx, clusterConfig)).To(Succeed()) + rawClient, err := ctl.NewRawClient(clusterConfig) Expect(err).NotTo(HaveOccurred()) return rawClient } diff --git a/pkg/actions/addon/addon.go b/pkg/actions/addon/addon.go index 51a3f0ef21..a3056aa7cb 100644 --- a/pkg/actions/addon/addon.go +++ b/pkg/actions/addon/addon.go @@ -37,12 +37,13 @@ type StackManager interface { type CreateClientSet func() (kubeclient.Interface, error) type Manager struct { - clusterConfig *api.ClusterConfig - eksAPI awsapi.EKS - withOIDC bool - oidcManager *iamoidc.OpenIDConnectManager - stackManager StackManager - createClientSet CreateClientSet + clusterConfig *api.ClusterConfig + eksAPI awsapi.EKS + withOIDC bool + oidcManager *iamoidc.OpenIDConnectManager + stackManager StackManager + createClientSet CreateClientSet + DisableAWSNodePatch bool } func New(clusterConfig *api.ClusterConfig, eksAPI awsapi.EKS, stackManager StackManager, withOIDC bool, oidcManager *iamoidc.OpenIDConnectManager, createClientSet CreateClientSet) (*Manager, error) { diff --git a/pkg/actions/addon/create.go b/pkg/actions/addon/create.go index 4496494fec..ac3109c606 100644 --- a/pkg/actions/addon/create.go +++ b/pkg/actions/addon/create.go @@ -242,7 +242,7 @@ func (a *Manager) Create(ctx context.Context, addon *api.Addon, iamRoleCreator I logger.Warning(IAMPermissionsNotRequiredWarning(addon.Name)) } - if addon.CanonicalName() == api.VPCCNIAddon { + if !a.DisableAWSNodePatch && addon.CanonicalName() == api.VPCCNIAddon { logger.Debug("patching AWS node") err := a.patchAWSNodeSA(ctx) if err != nil { diff --git a/pkg/actions/addon/tasks.go b/pkg/actions/addon/tasks.go index 800b301a75..f4478c8620 100644 --- a/pkg/actions/addon/tasks.go +++ b/pkg/actions/addon/tasks.go @@ -3,6 +3,7 @@ package addon import ( "context" "fmt" + "slices" "strings" "time" @@ -13,43 +14,105 @@ import ( api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/eks" + iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" "github.com/weaveworks/eksctl/pkg/utils/tasks" ) -func CreateAddonTasks(ctx context.Context, cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvider, iamRoleCreator IAMRoleCreator, forceAll bool, timeout time.Duration) (*tasks.TaskTree, *tasks.TaskTree) { - preTasks := &tasks.TaskTree{Parallel: false} - postTasks := &tasks.TaskTree{Parallel: false} - var preAddons []*api.Addon - var postAddons []*api.Addon - for _, addon := range cfg.Addons { - if strings.EqualFold(addon.Name, api.VPCCNIAddon) || - strings.EqualFold(addon.Name, api.PodIdentityAgentAddon) { +var knownAddons = map[string]struct { + IsDefault bool + CreateBeforeNodeGroup bool +}{ + api.VPCCNIAddon: { + IsDefault: true, + CreateBeforeNodeGroup: true, + }, + api.KubeProxyAddon: { + IsDefault: true, + CreateBeforeNodeGroup: true, + }, + api.CoreDNSAddon: { + IsDefault: true, + CreateBeforeNodeGroup: true, + }, + api.PodIdentityAgentAddon: { + CreateBeforeNodeGroup: true, + }, + api.AWSEBSCSIDriverAddon: {}, + api.AWSEFSCSIDriverAddon: {}, +} + +func CreateAddonTasks(ctx context.Context, cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvider, iamRoleCreator IAMRoleCreator, forceAll bool, timeout time.Duration) (*tasks.TaskTree, *tasks.TaskTree, *tasks.GenericTask, []string) { + var addons []*api.Addon + var autoDefaultAddonNames []string + if !cfg.AddonsConfig.DisableDefaultAddons { + addons = make([]*api.Addon, len(cfg.Addons)) + copy(addons, cfg.Addons) + + for addonName, addonInfo := range knownAddons { + if addonInfo.IsDefault && !slices.ContainsFunc(cfg.Addons, func(a *api.Addon) bool { + return strings.EqualFold(a.Name, addonName) + }) { + addons = append(addons, &api.Addon{Name: addonName}) + autoDefaultAddonNames = append(autoDefaultAddonNames, addonName) + } + } + } else { + addons = cfg.Addons + } + + var ( + preAddons []*api.Addon + postAddons []*api.Addon + ) + var vpcCNIAddon *api.Addon + for _, addon := range addons { + addonInfo, ok := knownAddons[addon.Name] + if ok && addonInfo.CreateBeforeNodeGroup { preAddons = append(preAddons, addon) } else { postAddons = append(postAddons, addon) } + if addon.Name == api.VPCCNIAddon { + vpcCNIAddon = addon + } } + preTasks := &tasks.TaskTree{Parallel: false} + postTasks := &tasks.TaskTree{Parallel: false} - preAddonsTask := createAddonTask{ - info: "create addons", - addons: preAddons, - ctx: ctx, - cfg: cfg, - clusterProvider: clusterProvider, - forceAll: forceAll, - timeout: timeout, - wait: false, - iamRoleCreator: iamRoleCreator, + makeAddonTask := func(addons []*api.Addon, wait bool) *createAddonTask { + return &createAddonTask{ + info: "create addons", + addons: addons, + ctx: ctx, + cfg: cfg, + clusterProvider: clusterProvider, + forceAll: forceAll, + timeout: timeout, + wait: wait, + iamRoleCreator: iamRoleCreator, + } } - preTasks.Append(&preAddonsTask) - - postAddonsTask := preAddonsTask - postAddonsTask.addons = postAddons - postAddonsTask.wait = cfg.HasNodes() - postTasks.Append(&postAddonsTask) - - return preTasks, postTasks + if len(preAddons) > 0 { + preTasks.Append(makeAddonTask(preAddons, false)) + } + if len(postAddons) > 0 { + postTasks.Append(makeAddonTask(postAddons, cfg.HasNodes())) + } + var updateVPCCNI *tasks.GenericTask + if vpcCNIAddon != nil && api.IsEnabled(cfg.IAM.WithOIDC) { + updateVPCCNI = &tasks.GenericTask{ + Description: "update VPC CNI to use IRSA if required", + Doer: func() error { + addonManager, err := createAddonManager(ctx, clusterProvider, cfg) + if err != nil { + return err + } + return addonManager.Update(ctx, vpcCNIAddon, nil, clusterProvider.AWSProvider.WaitTimeout()) + }, + } + } + return preTasks, postTasks, updateVPCCNI, autoDefaultAddonNames } type createAddonTask struct { @@ -68,25 +131,12 @@ type createAddonTask struct { func (t *createAddonTask) Describe() string { return t.info } func (t *createAddonTask) Do(errorCh chan error) error { - oidc, err := t.clusterProvider.NewOpenIDConnectManager(t.ctx, t.cfg) - if err != nil { - return err - } - - oidcProviderExists, err := oidc.CheckProviderExists(t.ctx) - if err != nil { - return err - } - - stackManager := t.clusterProvider.NewStackManager(t.cfg) - - addonManager, err := New(t.cfg, t.clusterProvider.AWSProvider.EKS(), stackManager, oidcProviderExists, oidc, func() (kubernetes.Interface, error) { - return t.clusterProvider.NewStdClientSet(t.cfg) - }) + addonManager, err := createAddonManager(t.ctx, t.clusterProvider, t.cfg) if err != nil { return err } + addonManager.DisableAWSNodePatch = true // always install EKS Pod Identity Agent Addon first, if present, // as other addons might require IAM permissions for _, a := range t.addons { @@ -96,6 +146,9 @@ func (t *createAddonTask) Do(errorCh chan error) error { if t.forceAll { a.Force = true } + if !t.wait { + t.timeout = 0 + } err := addonManager.Create(t.ctx, a, t.iamRoleCreator, t.timeout) if err != nil { go func() { @@ -112,6 +165,9 @@ func (t *createAddonTask) Do(errorCh chan error) error { if t.forceAll { a.Force = true } + if !t.wait { + t.timeout = 0 + } err := addonManager.Create(t.ctx, a, t.iamRoleCreator, t.timeout) if err != nil { go func() { @@ -127,6 +183,30 @@ func (t *createAddonTask) Do(errorCh chan error) error { return nil } +func createAddonManager(ctx context.Context, clusterProvider *eks.ClusterProvider, cfg *api.ClusterConfig) (*Manager, error) { + var ( + oidc *iamoidc.OpenIDConnectManager + oidcProviderExists bool + ) + if api.IsEnabled(cfg.IAM.WithOIDC) { + var err error + oidc, err = clusterProvider.NewOpenIDConnectManager(ctx, cfg) + if err != nil { + return nil, err + } + oidcProviderExists, err = oidc.CheckProviderExists(ctx) + if err != nil { + return nil, err + } + } + + stackManager := clusterProvider.NewStackManager(cfg) + + return New(cfg, clusterProvider.AWSProvider.EKS(), stackManager, oidcProviderExists, oidc, func() (kubernetes.Interface, error) { + return clusterProvider.NewStdClientSet(cfg) + }) +} + type deleteAddonIAMTask struct { ctx context.Context info string diff --git a/pkg/actions/cluster/get_test.go b/pkg/actions/cluster/get_test.go index d015bb5c0e..fe8c908703 100644 --- a/pkg/actions/cluster/get_test.go +++ b/pkg/actions/cluster/get_test.go @@ -49,7 +49,7 @@ var _ = Describe("Get", func() { intialProvider.MockEKS().On("ListClusters", mock.Anything, &awseks.ListClustersInput{ MaxResults: aws.Int32(100), Include: []string{"all"}, - }).Return(&awseks.ListClustersOutput{ + }, mock.Anything).Return(&awseks.ListClustersOutput{ Clusters: []string{"cluster1", "cluster2", "cluster3"}, }, nil) @@ -110,7 +110,7 @@ var _ = Describe("Get", func() { intialProvider.MockEKS().On("ListClusters", mock.Anything, &awseks.ListClustersInput{ MaxResults: aws.Int32(100), Include: []string{"all"}, - }).Return(nil, fmt.Errorf("foo")) + }, mock.Anything).Return(nil, fmt.Errorf("foo")) }) It("errors", func() { @@ -158,14 +158,14 @@ var _ = Describe("Get", func() { providerRegion1.MockEKS().On("ListClusters", mock.Anything, &awseks.ListClustersInput{ MaxResults: aws.Int32(100), Include: []string{"all"}, - }).Return(&awseks.ListClustersOutput{ + }, mock.Anything).Return(&awseks.ListClustersOutput{ Clusters: []string{"cluster1"}, }, nil) providerRegion2.MockEKS().On("ListClusters", mock.Anything, &awseks.ListClustersInput{ MaxResults: aws.Int32(100), Include: []string{"all"}, - }).Return(&awseks.ListClustersOutput{ + }, mock.Anything).Return(&awseks.ListClustersOutput{ Clusters: []string{"cluster2"}, }, nil) @@ -242,7 +242,7 @@ var _ = Describe("Get", func() { providerRegion1.MockEKS().On("ListClusters", mock.Anything, &awseks.ListClustersInput{ MaxResults: aws.Int32(100), Include: []string{"all"}, - }).Return(&awseks.ListClustersOutput{ + }, mock.Anything).Return(&awseks.ListClustersOutput{ Clusters: []string{"cluster1"}, }, nil) diff --git a/pkg/actions/cluster/owned_test.go b/pkg/actions/cluster/owned_test.go index a84fcefb22..3c95a5649f 100644 --- a/pkg/actions/cluster/owned_test.go +++ b/pkg/actions/cluster/owned_test.go @@ -97,7 +97,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.NewTasksToDeleteClusterWithNodeGroupsReturns(&tasks.TaskTree{ Tasks: []tasks.Task{&tasks.GenericTask{Doer: func() error { @@ -174,7 +174,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.NewTasksToDeleteClusterWithNodeGroupsReturns(&tasks.TaskTree{ Tasks: []tasks.Task{}, @@ -239,7 +239,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.NewTasksToDeleteClusterWithNodeGroupsReturns(&tasks.TaskTree{ Tasks: []tasks.Task{}, @@ -306,7 +306,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.NewTasksToDeleteClusterWithNodeGroupsReturns(&tasks.TaskTree{ Tasks: []tasks.Task{&tasks.GenericTask{Doer: func() error { diff --git a/pkg/actions/cluster/unowned_test.go b/pkg/actions/cluster/unowned_test.go index 53a13ce409..f4bf5ee3ab 100644 --- a/pkg/actions/cluster/unowned_test.go +++ b/pkg/actions/cluster/unowned_test.go @@ -116,7 +116,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.GetFargateStackReturns(&types.Stack{StackName: aws.String("fargate-role")}, nil) fakeStackManager.DeleteStackBySpecReturns(nil, nil) @@ -205,7 +205,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.GetFargateStackReturns(nil, nil) fakeStackManager.DeleteStackBySpecReturns(nil, nil) @@ -308,7 +308,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) fakeStackManager.GetFargateStackReturns(nil, nil) fakeStackManager.DeleteStackBySpecReturns(nil, nil) @@ -391,7 +391,7 @@ var _ = Describe("Delete", func() { p.MockEC2().On("DescribeKeyPairs", mock.Anything, mock.Anything).Return(&ec2.DescribeKeyPairsOutput{}, nil) - p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) + p.MockEC2().On("DescribeSecurityGroups", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSecurityGroupsOutput{}, nil) p.MockEKS().On("ListNodegroups", mock.Anything, mock.Anything).Return(&awseks.ListNodegroupsOutput{ Nodegroups: []string{"ng-1", "ng-2"}, diff --git a/pkg/actions/cluster/upgrade.go b/pkg/actions/cluster/upgrade.go index 20293fa384..8a2fb6d2c3 100644 --- a/pkg/actions/cluster/upgrade.go +++ b/pkg/actions/cluster/upgrade.go @@ -132,6 +132,8 @@ func getNextVersion(currentVersion string) (string, error) { return api.Version1_30, nil case api.Version1_30: return api.Version1_31, nil + case api.Version1_31: + return api.Version1_32, nil default: // version of control plane is not known to us, maybe we are just too old... return "", fmt.Errorf("control plane version %q is not known to this version of eksctl, try to upgrade eksctl first", currentVersion) diff --git a/pkg/actions/cluster/upgrade_test.go b/pkg/actions/cluster/upgrade_test.go index cc998cc957..50cac69a2e 100644 --- a/pkg/actions/cluster/upgrade_test.go +++ b/pkg/actions/cluster/upgrade_test.go @@ -86,9 +86,9 @@ var _ = Describe("upgrade cluster", func() { }), Entry("fails when the version is still not supported", upgradeCase{ - givenVersion: "1.31", + givenVersion: "1.32", eksVersion: api.LatestVersion, - expectedErrorText: "control plane version \"1.31\" is not known to this version of eksctl", + expectedErrorText: "control plane version \"1.32\" is not known to this version of eksctl", }), ) }) diff --git a/pkg/actions/nodegroup/create.go b/pkg/actions/nodegroup/create.go index b533980039..670df34e3e 100644 --- a/pkg/actions/nodegroup/create.go +++ b/pkg/actions/nodegroup/create.go @@ -39,6 +39,7 @@ type CreateOpts struct { DryRunSettings DryRunSettings SkipOutdatedAddonsCheck bool ConfigFileProvided bool + Parallelism int } type DryRunSettings struct { @@ -80,9 +81,11 @@ func (m *Manager) Create(ctx context.Context, options CreateOpts, nodegroupFilte return errors.Wrapf(err, "loading VPC spec for cluster %q", meta.Name) } isOwnedCluster = false - skipEgressRules, err = validateSecurityGroup(ctx, ctl.AWSProvider.EC2(), cfg.VPC.SecurityGroup) - if err != nil { - return err + if len(cfg.NodeGroups) > 0 { + skipEgressRules, err = validateSecurityGroup(ctx, ctl.AWSProvider.EC2(), cfg.VPC.SecurityGroup) + if err != nil { + return err + } } default: @@ -172,7 +175,7 @@ func (m *Manager) Create(ctx context.Context, options CreateOpts, nodegroupFilte return cmdutils.PrintNodeGroupDryRunConfig(clusterConfigCopy, options.DryRunSettings.OutStream) } - if err := m.nodeCreationTasks(ctx, isOwnedCluster, skipEgressRules, options.UpdateAuthConfigMap); err != nil { + if err := m.nodeCreationTasks(ctx, isOwnedCluster, skipEgressRules, options.UpdateAuthConfigMap, options.Parallelism); err != nil { return err } @@ -204,7 +207,7 @@ func makeOutpostsService(clusterConfig *api.ClusterConfig, provider api.ClusterP } } -func (m *Manager) nodeCreationTasks(ctx context.Context, isOwnedCluster, skipEgressRules bool, updateAuthConfigMap *bool) error { +func (m *Manager) nodeCreationTasks(ctx context.Context, isOwnedCluster, skipEgressRules bool, updateAuthConfigMap *bool, parallelism int) error { cfg := m.cfg meta := cfg.Metadata @@ -260,10 +263,10 @@ func (m *Manager) nodeCreationTasks(ctx context.Context, isOwnedCluster, skipEgr } disableAccessEntryCreation := !m.accessEntry.IsEnabled() || updateAuthConfigMap != nil if nodeGroupTasks := m.stackManager.NewUnmanagedNodeGroupTask(ctx, cfg.NodeGroups, !awsNodeUsesIRSA, skipEgressRules, - disableAccessEntryCreation, vpcImporter); nodeGroupTasks.Len() > 0 { + disableAccessEntryCreation, vpcImporter, parallelism); nodeGroupTasks.Len() > 0 { allNodeGroupTasks.Append(nodeGroupTasks) } - managedTasks := m.stackManager.NewManagedNodeGroupTask(ctx, cfg.ManagedNodeGroups, !awsNodeUsesIRSA, vpcImporter) + managedTasks := m.stackManager.NewManagedNodeGroupTask(ctx, cfg.ManagedNodeGroups, !awsNodeUsesIRSA, vpcImporter, parallelism) if managedTasks.Len() > 0 { allNodeGroupTasks.Append(managedTasks) } @@ -315,7 +318,7 @@ func (m *Manager) postNodeCreationTasks(ctx context.Context, clientSet kubernete if (!m.accessEntry.IsEnabled() && !api.IsDisabled(options.UpdateAuthConfigMap)) || // if explicitly requested by the user api.IsEnabled(options.UpdateAuthConfigMap) { - if err := eks.UpdateAuthConfigMap(ctx, m.cfg.NodeGroups, clientSet); err != nil { + if err := eks.UpdateAuthConfigMap(m.cfg.NodeGroups, clientSet); err != nil { return err } } diff --git a/pkg/actions/nodegroup/create_test.go b/pkg/actions/nodegroup/create_test.go index 0d511089d7..23a2aafce4 100644 --- a/pkg/actions/nodegroup/create_test.go +++ b/pkg/actions/nodegroup/create_test.go @@ -77,11 +77,11 @@ type stackManagerDelegate struct { ngTaskCreator nodeGroupTaskCreator } -func (s *stackManagerDelegate) NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, vpcImporter vpc.Importer) *tasks.TaskTree { +func (s *stackManagerDelegate) NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, vpcImporter vpc.Importer, nodeGroupParallelism int) *tasks.TaskTree { return s.ngTaskCreator.NewUnmanagedNodeGroupTask(ctx, nodeGroups, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation, vpcImporter) } -func (s *stackManagerDelegate) NewManagedNodeGroupTask(context.Context, []*api.ManagedNodeGroup, bool, vpc.Importer) *tasks.TaskTree { +func (s *stackManagerDelegate) NewManagedNodeGroupTask(context.Context, []*api.ManagedNodeGroup, bool, vpc.Importer, int) *tasks.TaskTree { return nil } @@ -168,7 +168,7 @@ var _ = DescribeTable("Create", func(t ngEntry) { mockCalls: func(m mockCalls) { m.kubeProvider.NewRawClientReturns(&kubernetes.RawClient{}, nil) m.kubeProvider.ServerVersionReturns("1.17", nil) - m.mockProvider.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + m.mockProvider.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cftypes.StackSummary{ { StackName: aws.String("eksctl-my-cluster-cluster"), @@ -854,7 +854,7 @@ func mockProviderWithVPCSubnets(p *mockprovider.MockProvider, subnets *vpcSubnet } func mockProviderWithConfig(p *mockprovider.MockProvider, describeStacksOutput []cftypes.Output, subnets *vpcSubnets, vpcConfigRes *ekstypes.VpcConfigResponse, outpostConfig *ekstypes.OutpostConfigResponse, accessConfig *ekstypes.AccessConfigResponse) { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cftypes.StackSummary{ { StackName: aws.String("eksctl-my-cluster-cluster"), @@ -986,7 +986,7 @@ func mockProviderWithConfig(p *mockprovider.MockProvider, describeStacksOutput [ if vpcID == "" { mp.MockEC2().On("DescribeSubnets", mock.Anything, &ec2.DescribeSubnetsInput{ SubnetIds: subnetIDs, - }).Return(&ec2.DescribeSubnetsOutput{Subnets: subnets}, nil) + }, mock.Anything).Return(&ec2.DescribeSubnetsOutput{Subnets: subnets}, nil) return } mp.MockEC2().On("DescribeSubnets", mock.Anything, &ec2.DescribeSubnetsInput{ @@ -996,7 +996,7 @@ func mockProviderWithConfig(p *mockprovider.MockProvider, describeStacksOutput [ Values: []string{vpcID}, }, }, - }).Return(&ec2.DescribeSubnetsOutput{Subnets: subnets}, nil) + }, mock.Anything).Return(&ec2.DescribeSubnetsOutput{Subnets: subnets}, nil) } mockDescribeSubnets(p, "", subnets.publicIDs) @@ -1021,7 +1021,7 @@ func mockProviderWithConfig(p *mockprovider.MockProvider, describeStacksOutput [ func mockProviderForUnownedCluster(p *mockprovider.MockProvider, k *eksfakes.FakeKubeProvider, extraSGRules ...ec2types.SecurityGroupRule) { k.NewRawClientReturns(&kubernetes.RawClient{}, nil) k.ServerVersionReturns("1.27", nil) - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cftypes.StackSummary{ { StackName: aws.String("eksctl-my-cluster-cluster"), @@ -1052,7 +1052,7 @@ func mockProviderForUnownedCluster(p *mockprovider.MockProvider, k *eksfakes.Fak }, }, }, nil) - p.MockEC2().On("DescribeSubnets", mock.Anything, mock.Anything).Return(&ec2.DescribeSubnetsOutput{ + p.MockEC2().On("DescribeSubnets", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { SubnetId: aws.String("subnet-custom1"), @@ -1076,7 +1076,7 @@ func mockProviderForUnownedCluster(p *mockprovider.MockProvider, k *eksfakes.Fak } filter := input.Filters[0] return *filter.Name == "group-id" && len(filter.Values) == 1 && filter.Values[0] == *sgID - })).Return(&ec2.DescribeSecurityGroupRulesOutput{ + }), mock.Anything).Return(&ec2.DescribeSecurityGroupRulesOutput{ SecurityGroupRules: append([]ec2types.SecurityGroupRule{ { Description: aws.String("Allow control plane to communicate with worker nodes in group ng-1 (kubelet and workload TCP ports"), @@ -1135,4 +1135,9 @@ func makeUnownedClusterConfig(clusterConfig *api.ClusterConfig) { }, }, } + clusterConfig.NodeGroups = append(clusterConfig.NodeGroups, &api.NodeGroup{ + NodeGroupBase: &api.NodeGroupBase{ + Name: "ng", + }, + }) } diff --git a/pkg/actions/nodegroup/drain.go b/pkg/actions/nodegroup/drain.go index 381822c172..c284ef5c11 100644 --- a/pkg/actions/nodegroup/drain.go +++ b/pkg/actions/nodegroup/drain.go @@ -58,6 +58,6 @@ func (d *Drainer) Drain(ctx context.Context, input *DrainInput) error { func waitForAllRoutinesToFinish(ctx context.Context, sem *semaphore.Weighted, size int64) { if err := sem.Acquire(ctx, size); err != nil { - logger.Critical("failed to acquire semaphore while waiting for all routines to finish: %w", err) + logger.Critical("failed to acquire semaphore while waiting for all routines to finish: %v", err) } } diff --git a/pkg/actions/nodegroup/testdata/al2-updated-template.json b/pkg/actions/nodegroup/testdata/al2-updated-template.json index 6d93868fe0..f8672102b1 100644 --- a/pkg/actions/nodegroup/testdata/al2-updated-template.json +++ b/pkg/actions/nodegroup/testdata/al2-updated-template.json @@ -140,7 +140,7 @@ ] }, "NodegroupName": "amazonlinux2", - "ReleaseVersion": "1.30-20201212", + "ReleaseVersion": "1.31-20201212", "ScalingConfig": { "DesiredSize": 4, "MaxSize": 4, diff --git a/pkg/actions/podidentityassociation/addon_migrator_test.go b/pkg/actions/podidentityassociation/addon_migrator_test.go index 7bb1fbe7a3..d797fe2c7a 100644 --- a/pkg/actions/podidentityassociation/addon_migrator_test.go +++ b/pkg/actions/podidentityassociation/addon_migrator_test.go @@ -46,7 +46,7 @@ var _ = Describe("Addon Migration", func() { mockAddonCalls := func(eksAddonsAPI *mocksv2.EKS) { eksAddonsAPI.On("ListAddons", mock.Anything, &eks.ListAddonsInput{ ClusterName: aws.String(clusterName), - }).Return(&eks.ListAddonsOutput{ + }, mock.Anything).Return(&eks.ListAddonsOutput{ Addons: []string{"vpc-cni"}, }, nil) eksAddonsAPI.On("DescribeAddon", mock.Anything, &eks.DescribeAddonInput{ diff --git a/pkg/actions/podidentityassociation/migrator_test.go b/pkg/actions/podidentityassociation/migrator_test.go index bb54a7a095..b1adf80e7a 100644 --- a/pkg/actions/podidentityassociation/migrator_test.go +++ b/pkg/actions/podidentityassociation/migrator_test.go @@ -96,7 +96,7 @@ var _ = Describe("Create", func() { mockProvider = mockprovider.NewMockProvider() mockProvider.MockEKS().On("ListAddons", mock.Anything, &awseks.ListAddonsInput{ ClusterName: aws.String(clusterName), - }).Return(&awseks.ListAddonsOutput{}, nil) + }, mock.Anything).Return(&awseks.ListAddonsOutput{}, nil) if e.mockEKS != nil { e.mockEKS(mockProvider) } diff --git a/pkg/actions/podidentityassociation/updater.go b/pkg/actions/podidentityassociation/updater.go index 22ab2d95ab..967451a415 100644 --- a/pkg/actions/podidentityassociation/updater.go +++ b/pkg/actions/podidentityassociation/updater.go @@ -136,9 +136,14 @@ func (u *Updater) makeUpdate(ctx context.Context, pia api.PodIdentityAssociation case 0: return nil, errors.New(notFoundErrMsg) case 1: + association := output.Associations[0] + if association.OwnerArn != nil { + return nil, fmt.Errorf("cannot update podidentityassociation %s as it is in use by addon %s; "+ + "please use `eksctl update addon` instead", pia.NameString(), *association.OwnerArn) + } describeOutput, err := u.APIUpdater.DescribePodIdentityAssociation(ctx, &eks.DescribePodIdentityAssociationInput{ ClusterName: aws.String(u.ClusterName), - AssociationId: output.Associations[0].AssociationId, + AssociationId: association.AssociationId, }) if err != nil { return nil, fmt.Errorf("error describing pod identity association: %w", err) diff --git a/pkg/addons/assets/efa-device-plugin.yaml b/pkg/addons/assets/efa-device-plugin.yaml index 71f2dc15e4..7227d5ff62 100644 --- a/pkg/addons/assets/efa-device-plugin.yaml +++ b/pkg/addons/assets/efa-device-plugin.yaml @@ -37,7 +37,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: "beta.kubernetes.io/instance-type" + - key: "node.kubernetes.io/instance-type" operator: In values: - c5n.18xlarge @@ -62,6 +62,9 @@ spec: - g6.24xlarge - g6.48xlarge - hpc6a.48xlarge + - hpc7g.16xlarge + - hpc7g.8xlarge + - hpc7g.4xlarge - i3en.12xlarge - i3en.24xlarge - i3en.metal @@ -124,6 +127,9 @@ spec: - g6.24xlarge - g6.48xlarge - hpc6a.48xlarge + - hpc7g.16xlarge + - hpc7g.8xlarge + - hpc7g.4xlarge - i3en.12xlarge - i3en.24xlarge - i3en.metal @@ -165,7 +171,6 @@ spec: - image: "%s.dkr.ecr.%s.%s/eks/aws-efa-k8s-device-plugin:v0.3.3" name: aws-efa-k8s-device-plugin securityContext: - runAsNonRoot: true allowPrivilegeEscalation: false capabilities: drop: ["ALL"] @@ -176,3 +181,4 @@ spec: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins + diff --git a/pkg/addons/assets/nvidia-device-plugin.yaml b/pkg/addons/assets/nvidia-device-plugin.yaml index 20cf248d02..681c23de21 100644 --- a/pkg/addons/assets/nvidia-device-plugin.yaml +++ b/pkg/addons/assets/nvidia-device-plugin.yaml @@ -25,18 +25,10 @@ spec: type: RollingUpdate template: metadata: - # This annotation is deprecated. Kept here for backward compatibility - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" labels: name: nvidia-device-plugin-ds spec: tolerations: - # This toleration is deprecated. Kept here for backward compatibility - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - - key: CriticalAddonsOnly - operator: Exists - key: nvidia.com/gpu operator: Exists effect: NoSchedule @@ -46,18 +38,19 @@ spec: # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ priorityClassName: "system-node-critical" containers: - - image: nvcr.io/nvidia/k8s-device-plugin:v0.9.0 + - image: nvcr.io/nvidia/k8s-device-plugin:v0.16.0 name: nvidia-device-plugin-ctr - args: ["--fail-on-init-error=false"] + env: + - name: FAIL_ON_INIT_ERROR + value: "false" securityContext: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - volumes: - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/pkg/addons/assets/scripts/update_nvidia_device_plugin.sh b/pkg/addons/assets/scripts/update_nvidia_device_plugin.sh new file mode 100755 index 0000000000..1ae0da602e --- /dev/null +++ b/pkg/addons/assets/scripts/update_nvidia_device_plugin.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +get_latest_release_tag() { + curl -sL https://api.github.com/repos/NVIDIA/k8s-device-plugin/releases/latest | jq -r '.tag_name' +} + +latest_release_tag=$(get_latest_release_tag) + +# Check if the latest release tag was found +if [ -z "$latest_release_tag" ]; then + echo "Could not find the latest release tag." + exit 1 +fi + +# If running in GitHub Actions, export the release tag for use in the workflow +if [ "$GITHUB_ACTIONS" = "true" ]; then + echo "LATEST_RELEASE_TAG= to $latest_release_tag" >> $GITHUB_ENV +else + echo "Found the latest release tag: $latest_release_tag" +fi + +assets_addons_dir="pkg/addons/assets" + +curl -sL "https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/$latest_release_tag/deployments/static/nvidia-device-plugin.yml" -o "$assets_addons_dir/nvidia-device-plugin.yaml" + + +# Check if the download was successful +if [ $? -eq 0 ]; then + echo "Downloaded the latest NVIDIA device plugin manifest to $assets_addons_dir/nvidia-device-plugin.yaml" +else + echo "Failed to download the NVIDIA device plugin manifest." + exit 1 +fi diff --git a/pkg/addons/default/addons.go b/pkg/addons/default/addons.go index 76036996f3..246c2452bf 100644 --- a/pkg/addons/default/addons.go +++ b/pkg/addons/default/addons.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/eks" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -11,15 +13,20 @@ import ( "github.com/kris-nova/logger" "github.com/pkg/errors" - "github.com/weaveworks/eksctl/pkg/awsapi" "github.com/weaveworks/eksctl/pkg/kubernetes" ) type AddonInput struct { - RawClient kubernetes.RawClientInterface - EKSAPI awsapi.EKS - ControlPlaneVersion string - Region string + RawClient kubernetes.RawClientInterface + AddonVersionDescriber AddonVersionDescriber + ControlPlaneVersion string + Region string +} + +// AddonVersionDescriber describes the versions for an addon. +type AddonVersionDescriber interface { + // DescribeAddonVersions describes the versions for an addon. + DescribeAddonVersions(ctx context.Context, params *eks.DescribeAddonVersionsInput, optFns ...func(options *eks.Options)) (*eks.DescribeAddonVersionsOutput, error) } // DoAddonsSupportMultiArch checks if the coredns/kubeproxy/awsnode support multi arch nodegroups diff --git a/pkg/addons/default/assets/coredns-1.31.json b/pkg/addons/default/assets/coredns-1.31.json new file mode 100644 index 0000000000..42753c02bd --- /dev/null +++ b/pkg/addons/default/assets/coredns-1.31.json @@ -0,0 +1,379 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "annotations": { + "prometheus.io/port": "9153", + "prometheus.io/scrape": "true" + }, + "labels": { + "eks.amazonaws.com/component": "kube-dns", + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "kubernetes.io/name": "CoreDNS" + }, + "name": "kube-dns", + "namespace": "kube-system" + }, + "spec": { + "internalTrafficPolicy": "Cluster", + "ipFamilies": [ + "IPv4" + ], + "ipFamilyPolicy": "SingleStack", + "ports": [ + { + "name": "dns", + "port": 53, + "protocol": "UDP", + "targetPort": 53 + }, + { + "name": "dns-tcp", + "port": 53, + "protocol": "TCP", + "targetPort": 53 + }, + { + "name": "metrics", + "port": 9153, + "protocol": "TCP", + "targetPort": 9153 + } + ], + "selector": { + "k8s-app": "kube-dns" + }, + "sessionAffinity": "None", + "type": "ClusterIP" + } + }, + { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns" + }, + "name": "coredns", + "namespace": "kube-system" + } + }, + { + "apiVersion": "v1", + "data": { + "Corefile": ".:53 {\n errors\n health {\n lameduck 5s\n }\n ready\n kubernetes cluster.local in-addr.arpa ip6.arpa {\n pods insecure\n fallthrough in-addr.arpa ip6.arpa\n }\n prometheus :9153\n forward . /etc/resolv.conf\n cache 30\n loop\n reload\n loadbalance\n}\n" + }, + "kind": "ConfigMap", + "metadata": { + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns" + }, + "name": "coredns", + "namespace": "kube-system" + } + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": {}, + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns", + "kubernetes.io/name": "CoreDNS" + }, + "name": "coredns", + "namespace": "kube-system" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 2, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": "25%", + "maxUnavailable": 1 + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns" + } + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/os", + "operator": "In", + "values": [ + "linux" + ] + }, + { + "key": "kubernetes.io/arch", + "operator": "In", + "values": [ + "amd64", + "arm64" + ] + } + ] + } + ] + } + }, + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [ + { + "key": "k8s-app", + "operator": "In", + "values": [ + "kube-dns" + ] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 100 + } + ] + } + }, + "containers": [ + { + "args": [ + "-conf", + "/etc/coredns/Corefile" + ], + "image": "%s.dkr.ecr.%s.%s/eks/coredns:v1.11.3-eksbuild.1", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 5, + "httpGet": { + "path": "/health", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "name": "coredns", + "ports": [ + { + "containerPort": 53, + "name": "dns", + "protocol": "UDP" + }, + { + "containerPort": 53, + "name": "dns-tcp", + "protocol": "TCP" + }, + { + "containerPort": 9153, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/ready", + "port": 8181, + "scheme": "HTTP" + }, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "170Mi" + }, + "requests": { + "cpu": "100m", + "memory": "70Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "add": [ + "NET_BIND_SERVICE" + ], + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/etc/coredns", + "name": "config-volume", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "Default", + "priorityClassName": "system-cluster-critical", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "coredns", + "serviceAccountName": "coredns", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/control-plane" + }, + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + } + ], + "topologySpreadConstraints": [ + { + "labelSelector": { + "matchLabels": { + "k8s-app": "kube-dns" + } + }, + "maxSkew": 1, + "topologyKey": "topology.kubernetes.io/zone", + "whenUnsatisfiable": "ScheduleAnyway" + } + ], + "volumes": [ + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "Corefile", + "path": "Corefile" + } + ], + "name": "coredns" + }, + "name": "config-volume" + } + ] + } + } + } + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRole", + "metadata": { + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns", + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "system:coredns" + }, + "rules": [ + { + "apiGroups": [ + "" + ], + "resources": [ + "endpoints", + "services", + "pods", + "namespaces" + ], + "verbs": [ + "list", + "watch" + ] + }, + { + "apiGroups": [ + "" + ], + "resources": [ + "nodes" + ], + "verbs": [ + "get" + ] + }, + { + "apiGroups": [ + "discovery.k8s.io" + ], + "resources": [ + "endpointslices" + ], + "verbs": [ + "list", + "watch" + ] + } + ] + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "annotations": { + "rbac.authorization.kubernetes.io/autoupdate": "true" + }, + "labels": { + "eks.amazonaws.com/component": "coredns", + "k8s-app": "kube-dns", + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "system:coredns" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:coredns" + }, + "subjects": [ + { + "kind": "ServiceAccount", + "name": "coredns", + "namespace": "kube-system" + } + ] + } + ], + "kind": "List" +} diff --git a/pkg/addons/default/kube_proxy.go b/pkg/addons/default/kube_proxy.go index 8f992c4f31..2bf0778a5e 100644 --- a/pkg/addons/default/kube_proxy.go +++ b/pkg/addons/default/kube_proxy.go @@ -16,7 +16,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/weaveworks/eksctl/pkg/awsapi" "github.com/weaveworks/eksctl/pkg/kubernetes" "github.com/weaveworks/eksctl/pkg/printers" ) @@ -117,7 +116,7 @@ func addArm64NodeSelector(daemonSet *v1.DaemonSet) error { func getLatestKubeProxyImage(ctx context.Context, input AddonInput) (string, error) { defaultClusterVersion := generateImageVersionFromClusterVersion(input.ControlPlaneVersion) - latestEKSReportedVersion, err := getLatestImageVersionFromEKS(ctx, input.EKSAPI, input.ControlPlaneVersion) + latestEKSReportedVersion, err := getLatestImageVersionFromEKS(ctx, input.AddonVersionDescriber, input.ControlPlaneVersion) if err != nil { return "", err } @@ -151,7 +150,7 @@ func generateImageVersionFromClusterVersion(controlPlaneVersion string) string { return fmt.Sprintf("v%s-eksbuild.1", controlPlaneVersion) } -func getLatestImageVersionFromEKS(ctx context.Context, eksAPI awsapi.EKS, controlPlaneVersion string) (string, error) { +func getLatestImageVersionFromEKS(ctx context.Context, addonDescriber AddonVersionDescriber, controlPlaneVersion string) (string, error) { controlPlaneMajorMinor, err := versionWithOnlyMajorAndMinor(controlPlaneVersion) if err != nil { return "", err @@ -161,7 +160,7 @@ func getLatestImageVersionFromEKS(ctx context.Context, eksAPI awsapi.EKS, contro AddonName: aws.String(KubeProxy), } - addonInfos, err := eksAPI.DescribeAddonVersions(ctx, input) + addonInfos, err := addonDescriber.DescribeAddonVersions(ctx, input) if err != nil { return "", fmt.Errorf("failed to describe addon versions: %v", err) } diff --git a/pkg/addons/default/kube_proxy_test.go b/pkg/addons/default/kube_proxy_test.go index f1e05b59ba..f74c7077f0 100644 --- a/pkg/addons/default/kube_proxy_test.go +++ b/pkg/addons/default/kube_proxy_test.go @@ -36,10 +36,10 @@ var _ = Describe("KubeProxy", func() { clientSet = rawClient.ClientSet() mockProvider = mockprovider.NewMockProvider() input = da.AddonInput{ - RawClient: rawClient, - Region: "eu-west-1", - EKSAPI: mockProvider.EKS(), - ControlPlaneVersion: controlPlaneVersion, + RawClient: rawClient, + Region: "eu-west-1", + AddonVersionDescriber: mockProvider.EKS(), + ControlPlaneVersion: controlPlaneVersion, } }) diff --git a/pkg/addons/default/scripts/update_aws_node.sh b/pkg/addons/default/scripts/update_aws_node.sh index 6f17d9c96e..f29ac75cc4 100755 --- a/pkg/addons/default/scripts/update_aws_node.sh +++ b/pkg/addons/default/scripts/update_aws_node.sh @@ -9,12 +9,31 @@ get_latest_release_tag() { latest_release_tag=$(get_latest_release_tag) +# Check if the latest release tag was found +if [ -z "$latest_release_tag" ]; then + echo "Could not find the latest release tag." + exit 1 +fi + +# If running in GitHub Actions, export the release tag for use in the workflow +if [ "$GITHUB_ACTIONS" = "true" ]; then + echo "LATEST_RELEASE_TAG= to $latest_release_tag" >> $GITHUB_ENV +else + echo "Found the latest release tag: $latest_release_tag" +fi + default_addons_dir="pkg/addons/default" # Download the latest aws-k8s-cni.yaml file curl -sL "$base_url$latest_release_tag/config/master/aws-k8s-cni.yaml?raw=1" --output "$default_addons_dir/assets/aws-node.yaml" -echo "found latest release tag:" $latest_release_tag +# Check if the download was successful +if [ $? -eq 0 ]; then + echo "Downloaded the latest AWS Node manifest to $default_addons_dir/assets/aws-node.yaml" +else + echo "Failed to download the latest AWS Node manifest." + exit 1 +fi # Update the unit test file sed -i "s/expectedVersion = \"\(.*\)\"/expectedVersion = \"$latest_release_tag\"/g" "$default_addons_dir/aws_node_test.go" diff --git a/pkg/ami/api.go b/pkg/ami/api.go index 53812150f2..a19312fa21 100644 --- a/pkg/ami/api.go +++ b/pkg/ami/api.go @@ -19,14 +19,16 @@ import ( // Variations of image classes const ( ImageClassGeneral = iota - ImageClassGPU + ImageClassNvidia + ImageClassNeuron ImageClassARM ) // ImageClasses is a list of image class names var ImageClasses = []string{ "ImageClassGeneral", - "ImageClassGPU", + "ImageClassNvidia", + "ImageClassNeuron", "ImageClassARM", } diff --git a/pkg/ami/auto_resolver.go b/pkg/ami/auto_resolver.go index b0a49f4597..883490723a 100644 --- a/pkg/ami/auto_resolver.go +++ b/pkg/ami/auto_resolver.go @@ -25,11 +25,14 @@ func MakeImageSearchPatterns(version string) map[string]map[int]string { return map[string]map[int]string{ api.NodeImageFamilyAmazonLinux2023: { ImageClassGeneral: fmt.Sprintf("amazon-eks-node-al2023-x86_64-standard-%s-v*", version), + ImageClassNvidia: fmt.Sprintf("amazon-eks-node-al2023-x86_64-nvidia-*-%s-v*", version), + ImageClassNeuron: fmt.Sprintf("amazon-eks-node-al2023-x86_64-neuron-%s-v*", version), ImageClassARM: fmt.Sprintf("amazon-eks-node-al2023-arm64-standard-%s-v*", version), }, api.NodeImageFamilyAmazonLinux2: { ImageClassGeneral: fmt.Sprintf("amazon-eks-node-%s-v*", version), - ImageClassGPU: fmt.Sprintf("amazon-eks-gpu-node-%s-*", version), + ImageClassNvidia: fmt.Sprintf("amazon-eks-gpu-node-%s-*", version), + ImageClassNeuron: fmt.Sprintf("amazon-eks-gpu-node-%s-*", version), ImageClassARM: fmt.Sprintf("amazon-eks-arm64-node-%s-*", version), }, api.NodeImageFamilyUbuntuPro2204: { @@ -90,16 +93,22 @@ func (r *AutoResolver) Resolve(ctx context.Context, region, version, instanceTyp imageClasses := MakeImageSearchPatterns(version)[imageFamily] namePattern := imageClasses[ImageClassGeneral] - if instanceutils.IsGPUInstanceType(instanceType) { + var ok bool + switch { + case instanceutils.IsNvidiaInstanceType(instanceType): + namePattern, ok = imageClasses[ImageClassNvidia] + if !ok { + logger.Critical("image family %s doesn't support Nvidia GPU image class", imageFamily) + return "", NewErrFailedResolution(region, version, instanceType, imageFamily) + } + case instanceutils.IsNeuronInstanceType(instanceType): var ok bool - namePattern, ok = imageClasses[ImageClassGPU] + namePattern, ok = imageClasses[ImageClassNeuron] if !ok { - logger.Critical("image family %s doesn't support GPU image class", imageFamily) + logger.Critical("image family %s doesn't support Neuron GPU image class", imageFamily) return "", NewErrFailedResolution(region, version, instanceType, imageFamily) } - } - - if instanceutils.IsARMInstanceType(instanceType) { + case instanceutils.IsARMInstanceType(instanceType): var ok bool namePattern, ok = imageClasses[ImageClassARM] if !ok { diff --git a/pkg/ami/ssm_resolver.go b/pkg/ami/ssm_resolver.go index bbc147b88b..2020e53716 100644 --- a/pkg/ami/ssm_resolver.go +++ b/pkg/ami/ssm_resolver.go @@ -55,8 +55,8 @@ func MakeSSMParameterName(version, instanceType, imageFamily string) (string, er switch imageFamily { case api.NodeImageFamilyAmazonLinux2023: - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/%s/standard/recommended/%s", - version, utils.ToKebabCase(imageFamily), instanceEC2ArchName(instanceType), fieldName), nil + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/%s/%s/recommended/%s", + version, utils.ToKebabCase(imageFamily), instanceEC2ArchName(instanceType), imageType(imageFamily, instanceType, version), fieldName), nil case api.NodeImageFamilyAmazonLinux2: return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/recommended/%s", version, imageType(imageFamily, instanceType, version), fieldName), nil case api.NodeImageFamilyWindowsServer2019CoreContainer, @@ -75,9 +75,19 @@ func MakeSSMParameterName(version, instanceType, imageFamily string) (string, er return fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2022-English-%s-EKS_Optimized-%s/%s", windowsAmiType(imageFamily), version, fieldName), nil case api.NodeImageFamilyBottlerocket: return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/%s", imageType(imageFamily, instanceType, version), instanceEC2ArchName(instanceType), fieldName), nil - case api.NodeImageFamilyUbuntuPro2204, api.NodeImageFamilyUbuntu2204, api.NodeImageFamilyUbuntu2004, api.NodeImageFamilyUbuntu1804: - // FIXME: SSM lookup for Ubuntu EKS images is supported nowadays - return "", &UnsupportedQueryError{msg: fmt.Sprintf("SSM Parameter lookups for %s AMIs is not supported yet", imageFamily)} + case api.NodeImageFamilyUbuntu1804: + return "", &UnsupportedQueryError{msg: fmt.Sprintf("SSM Parameter lookups for %s AMIs is not supported", imageFamily)} + case api.NodeImageFamilyUbuntu2004, + api.NodeImageFamilyUbuntu2204, + api.NodeImageFamilyUbuntuPro2204: + if err := validateVersionForUbuntu(version, imageFamily); err != nil { + return "", err + } + eksProduct := "eks" + if imageFamily == api.NodeImageFamilyUbuntuPro2204 { + eksProduct = "eks-pro" + } + return fmt.Sprint("/aws/service/canonical/ubuntu/", eksProduct, "/", ubuntuReleaseName(imageFamily), "/", version, "/stable/current/", ubuntuArchName(instanceType), "/hvm/ebs-gp2/ami-id"), nil default: return "", fmt.Errorf("unknown image family %s", imageFamily) } @@ -92,6 +102,10 @@ func MakeManagedSSMParameterName(version string, amiType ekstypes.AMITypes) stri switch amiType { case ekstypes.AMITypesAl2023X8664Standard: return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/x86_64/standard/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)) + case ekstypes.AMITypesAl2023X8664Nvidia: + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/x86_64/nvidia/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)) + case ekstypes.AMITypesAl2023X8664Neuron: + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/x86_64/neuron/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)) case ekstypes.AMITypesAl2023Arm64Standard: return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/arm64/standard/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)) case ekstypes.AMITypesAl2X8664: @@ -118,9 +132,24 @@ func instanceEC2ArchName(instanceType string) string { return "x86_64" } +func ubuntuArchName(instanceType string) string { + if instanceutils.IsARMInstanceType(instanceType) { + return "arm64" + } + return "amd64" +} + func imageType(imageFamily, instanceType, version string) string { family := utils.ToKebabCase(imageFamily) switch imageFamily { + case api.NodeImageFamilyAmazonLinux2023: + if instanceutils.IsNvidiaInstanceType(instanceType) { + return "nvidia" + } + if instanceutils.IsNeuronInstanceType(instanceType) { + return "neuron" + } + return "standard" case api.NodeImageFamilyBottlerocket: if instanceutils.IsNvidiaInstanceType(instanceType) { return fmt.Sprintf("%s-%s", version, "nvidia") @@ -143,3 +172,52 @@ func windowsAmiType(imageFamily string) string { } return "Full" } + +func ubuntuReleaseName(imageFamily string) string { + switch imageFamily { + case api.NodeImageFamilyUbuntu2004: + return "20.04" + case api.NodeImageFamilyUbuntu2204, api.NodeImageFamilyUbuntuPro2204: + return "22.04" + default: + return "18.04" + } +} + +func validateVersionForUbuntu(version, imageFamily string) error { + switch imageFamily { + case api.NodeImageFamilyUbuntu2004: + var err error + supportsUbuntu := false + const minVersion = api.Version1_21 + const maxVersion = api.Version1_29 + supportsUbuntu, err = utils.IsMinVersion(minVersion, version) + if err != nil { + return err + } + if !supportsUbuntu { + return &UnsupportedQueryError{msg: fmt.Sprintf("%s requires EKS version greater or equal than %s and lower than %s", imageFamily, minVersion, maxVersion)} + } + supportsUbuntu, err = utils.IsMinVersion(version, maxVersion) + if err != nil { + return err + } + if !supportsUbuntu { + return &UnsupportedQueryError{msg: fmt.Sprintf("%s requires EKS version greater or equal than %s and lower than %s", imageFamily, minVersion, maxVersion)} + } + case api.NodeImageFamilyUbuntu2204, api.NodeImageFamilyUbuntuPro2204: + var err error + supportsUbuntu := false + const minVersion = api.Version1_29 + supportsUbuntu, err = utils.IsMinVersion(minVersion, version) + if err != nil { + return err + } + if !supportsUbuntu { + return &UnsupportedQueryError{msg: fmt.Sprintf("%s requires EKS version greater or equal than %s", imageFamily, minVersion)} + } + default: + return &UnsupportedQueryError{msg: fmt.Sprintf("SSM Parameter lookups for %s AMIs is not supported", imageFamily)} + } + return nil +} diff --git a/pkg/ami/ssm_resolver_test.go b/pkg/ami/ssm_resolver_test.go index fec0b84193..ffb23cdabf 100644 --- a/pkg/ami/ssm_resolver_test.go +++ b/pkg/ami/ssm_resolver_test.go @@ -2,6 +2,7 @@ package ami_test import ( "context" + "fmt" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -233,10 +234,11 @@ var _ = Describe("AMI Auto Resolution", func() { }) - Context("and Ubuntu family", func() { + Context("and Ubuntu1804 family", func() { BeforeEach(func() { p = mockprovider.NewMockProvider() - imageFamily = "Ubuntu2004" + instanceType = "t2.medium" + imageFamily = "Ubuntu1804" }) It("should return an error", func() { @@ -244,6 +246,200 @@ var _ = Describe("AMI Auto Resolution", func() { resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("SSM Parameter lookups for Ubuntu1804 AMIs is not supported")) + }) + + }) + + Context("and Ubuntu2004 family", func() { + BeforeEach(func() { + p = mockprovider.NewMockProvider() + instanceType = "t2.medium" + imageFamily = "Ubuntu2004" + }) + + DescribeTable("should return an error", + func(version string) { + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("Ubuntu2004 requires EKS version greater or equal than 1.21 and lower than 1.29")) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.20"), + Entry(nil, "1.30"), + ) + + DescribeTable("should return a valid AMI", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.21"), + Entry(nil, "1.22"), + Entry(nil, "1.23"), + Entry(nil, "1.24"), + Entry(nil, "1.25"), + Entry(nil, "1.26"), + Entry(nil, "1.27"), + Entry(nil, "1.28"), + Entry(nil, "1.29"), + ) + + Context("for arm instance type", func() { + BeforeEach(func() { + instanceType = "a1.large" + }) + DescribeTable("should return a valid AMI for arm64", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.21"), + Entry(nil, "1.22"), + Entry(nil, "1.23"), + Entry(nil, "1.24"), + Entry(nil, "1.25"), + Entry(nil, "1.26"), + Entry(nil, "1.27"), + Entry(nil, "1.28"), + Entry(nil, "1.29"), + ) + }) + }) + + Context("and Ubuntu2204 family", func() { + BeforeEach(func() { + p = mockprovider.NewMockProvider() + instanceType = "t2.medium" + imageFamily = "Ubuntu2204" + }) + + DescribeTable("should return an error", + func(version string) { + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("Ubuntu2204 requires EKS version greater or equal than 1.29")) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.21"), + Entry(nil, "1.28"), + ) + + DescribeTable("should return a valid AMI", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/22.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.29"), + Entry(nil, "1.30"), + Entry(nil, "1.31"), + ) + + Context("for arm instance type", func() { + BeforeEach(func() { + instanceType = "a1.large" + }) + DescribeTable("should return a valid AMI for arm64", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/22.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.29"), + Entry(nil, "1.30"), + Entry(nil, "1.31"), + ) + }) + }) + + Context("and UbuntuPro2204 family", func() { + BeforeEach(func() { + p = mockprovider.NewMockProvider() + instanceType = "t2.medium" + imageFamily = "UbuntuPro2204" + }) + + DescribeTable("should return an error", + func(version string) { + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("UbuntuPro2204 requires EKS version greater or equal than 1.29")) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.21"), + Entry(nil, "1.28"), + ) + + DescribeTable("should return a valid AMI", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks-pro/22.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.29"), + Entry(nil, "1.30"), + Entry(nil, "1.31"), + ) + + Context("for arm instance type", func() { + BeforeEach(func() { + instanceType = "a1.large" + }) + DescribeTable("should return a valid AMI for arm64", + func(version string) { + addMockGetParameter(p, fmt.Sprintf("/aws/service/canonical/ubuntu/eks-pro/22.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version), expectedAmi) + + resolver := NewSSMResolver(p.MockSSM()) + resolvedAmi, err = resolver.Resolve(context.Background(), region, version, instanceType, imageFamily) + + Expect(err).NotTo(HaveOccurred()) + Expect(p.MockSSM().AssertNumberOfCalls(GinkgoT(), "GetParameter", 1)).To(BeTrue()) + Expect(resolvedAmi).To(BeEquivalentTo(expectedAmi)) + }, + EntryDescription("When EKS version is %s"), + Entry(nil, "1.29"), + Entry(nil, "1.30"), + Entry(nil, "1.31"), + ) }) }) @@ -405,7 +601,9 @@ var _ = Describe("AMI Auto Resolution", func() { It("should support SSM parameter generation for all AMI types but Windows", func() { var eksAMIType ekstypes.AMITypes for _, amiType := range eksAMIType.Values() { - if amiType == ekstypes.AMITypesCustom || strings.HasPrefix(string(amiType), "WINDOWS_") { + if amiType == ekstypes.AMITypesCustom || strings.HasPrefix(string(amiType), "WINDOWS_") || + // TODO: remove this condition after adding support for AL2023 Nvidia and Neuron AMI types. + amiType == ekstypes.AMITypesAl2023X8664Nvidia || amiType == ekstypes.AMITypesAl2023X8664Neuron { continue } ssmParameterName := MakeManagedSSMParameterName(api.LatestVersion, amiType) diff --git a/pkg/apis/eksctl.io/v1alpha5/addon.go b/pkg/apis/eksctl.io/v1alpha5/addon.go index 084aa6844f..707572c0cb 100644 --- a/pkg/apis/eksctl.io/v1alpha5/addon.go +++ b/pkg/apis/eksctl.io/v1alpha5/addon.go @@ -9,6 +9,18 @@ import ( "sigs.k8s.io/yaml" ) +// Values for core addons +const ( + minimumVPCCNIVersionForIPv6 = "1.10.0" + + VPCCNIAddon = "vpc-cni" + KubeProxyAddon = "kube-proxy" + CoreDNSAddon = "coredns" + PodIdentityAgentAddon = "eks-pod-identity-agent" + AWSEBSCSIDriverAddon = "aws-ebs-csi-driver" + AWSEFSCSIDriverAddon = "aws-efs-csi-driver" +) + // Addon holds the EKS addon configuration type Addon struct { // +required @@ -64,6 +76,12 @@ type AddonsConfig struct { // for supported addons that require IAM permissions. // +optional AutoApplyPodIdentityAssociations bool `json:"autoApplyPodIdentityAssociations,omitempty"` + + // DisableDefaultAddons enables or disables creation of default networking addons when the cluster + // is created. + // By default, all default addons are installed as EKS addons. + // +optional + DisableDefaultAddons bool `json:"disableDefaultAddons,omitempty"` } func (a Addon) CanonicalName() string { diff --git a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json index 9ac5d65b13..28502cc46b 100755 --- a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json +++ b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json @@ -269,10 +269,17 @@ "description": "specifies whether to automatically apply pod identity associations for supported addons that require IAM permissions.", "x-intellij-html-description": "specifies whether to automatically apply pod identity associations for supported addons that require IAM permissions.", "default": "false" + }, + "disableDefaultAddons": { + "type": "boolean", + "description": "enables or disables creation of default networking addons when the cluster is created. By default, all default addons are installed as EKS addons.", + "x-intellij-html-description": "enables or disables creation of default networking addons when the cluster is created. By default, all default addons are installed as EKS addons.", + "default": "false" } }, "preferredOrder": [ - "autoApplyPodIdentityAssociations" + "autoApplyPodIdentityAssociations", + "disableDefaultAddons" ], "additionalProperties": false, "description": "holds the addons config.", @@ -769,8 +776,8 @@ }, "version": { "type": "string", - "description": "Valid variants are: `\"1.23\"`, `\"1.24\"`, `\"1.25\"`, `\"1.26\"`, `\"1.27\"`, `\"1.28\"`, `\"1.29\"`, `\"1.30\"` (default).", - "x-intellij-html-description": "Valid variants are: "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29", "1.30" (default).", + "description": "Valid variants are: `\"1.23\"`, `\"1.24\"`, `\"1.25\"`, `\"1.26\"`, `\"1.27\"`, `\"1.28\"`, `\"1.29\"`, `\"1.30\"` (default), `\"1.31\"`.", + "x-intellij-html-description": "Valid variants are: "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29", "1.30" (default), "1.31".", "default": "1.30", "enum": [ "1.23", @@ -780,7 +787,8 @@ "1.27", "1.28", "1.29", - "1.30" + "1.30", + "1.31" ] } }, diff --git a/pkg/apis/eksctl.io/v1alpha5/defaults.go b/pkg/apis/eksctl.io/v1alpha5/defaults.go index 6bb309927b..88fe169066 100644 --- a/pkg/apis/eksctl.io/v1alpha5/defaults.go +++ b/pkg/apis/eksctl.io/v1alpha5/defaults.go @@ -79,7 +79,7 @@ func SetClusterConfigDefaults(cfg *ClusterConfig) { // IAM SAs that need to be explicitly deleted. func IAMServiceAccountsWithImplicitServiceAccounts(cfg *ClusterConfig) []*ClusterIAMServiceAccount { serviceAccounts := cfg.IAM.ServiceAccounts - if IsEnabled(cfg.IAM.WithOIDC) && !vpcCNIAddonSpecified(cfg) { + if IsEnabled(cfg.IAM.WithOIDC) && !vpcCNIAddonSpecified(cfg) && !cfg.AddonsConfig.DisableDefaultAddons { var found bool for _, sa := range cfg.IAM.ServiceAccounts { found = found || (sa.Name == AWSNodeMeta.Name && sa.Namespace == AWSNodeMeta.Namespace) @@ -134,7 +134,8 @@ func SetManagedNodeGroupDefaults(ng *ManagedNodeGroup, meta *ClusterMeta, contro // When using custom AMIs, we want the user to explicitly specify AMI family. // Thus, we only set up default AMI family when no custom AMI is being used. if ng.AMIFamily == "" && ng.AMI == "" { - if isMinVer, _ := utils.IsMinVersion(Version1_30, meta.Version); isMinVer && !instanceutils.IsGPUInstanceType(ng.InstanceType) && + + if isMinVer, _ := utils.IsMinVersion(Version1_30, meta.Version); isMinVer && !instanceutils.IsARMGPUInstanceType(ng.InstanceType) { ng.AMIFamily = NodeImageFamilyAmazonLinux2023 } else { diff --git a/pkg/apis/eksctl.io/v1alpha5/defaults_test.go b/pkg/apis/eksctl.io/v1alpha5/defaults_test.go index a0d470275e..e47147e0dc 100644 --- a/pkg/apis/eksctl.io/v1alpha5/defaults_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/defaults_test.go @@ -392,6 +392,7 @@ var _ = Describe("ClusterConfig validation", func() { }, false) Expect(mng.AMIFamily).To(Equal(expectedAMIFamily)) }, + Entry("EKS 1.31 uses AL2023", "1.31", NodeImageFamilyAmazonLinux2023), Entry("EKS 1.30 uses AL2023", "1.30", NodeImageFamilyAmazonLinux2023), Entry("EKS 1.29 uses AL2", "1.29", NodeImageFamilyAmazonLinux2), Entry("EKS 1.28 uses AL2", "1.28", NodeImageFamilyAmazonLinux2), diff --git a/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go b/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go index 33796d387a..404105932b 100644 --- a/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go @@ -40,22 +40,16 @@ var _ = Describe("GPU instance support", func() { assertValidationError(e, api.ValidateManagedNodeGroup(0, mng)) }, Entry("AL2023 INF", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "inf1.xlarge", - expectUnsupportedErr: true, - instanceTypeName: "Inferentia", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "inf1.xlarge", }), Entry("AL2023 TRN", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "trn1.2xlarge", - expectUnsupportedErr: true, - instanceTypeName: "Trainium", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "trn1.2xlarge", }), Entry("AL2023 NVIDIA", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "g4dn.xlarge", - expectUnsupportedErr: true, - instanceTypeName: "GPU", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "g4dn.xlarge", }), Entry("AL2", gpuInstanceEntry{ gpuInstanceType: "asdf", @@ -107,22 +101,16 @@ var _ = Describe("GPU instance support", func() { }, Entry("AL2023 INF", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "inf1.xlarge", - expectUnsupportedErr: true, - instanceTypeName: "Inferentia", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "inf1.xlarge", }), Entry("AL2023 TRN", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "trn1.2xlarge", - expectUnsupportedErr: true, - instanceTypeName: "Trainium", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "trn1.2xlarge", }), Entry("AL2023 NVIDIA", gpuInstanceEntry{ - amiFamily: api.NodeImageFamilyAmazonLinux2023, - gpuInstanceType: "g4dn.xlarge", - expectUnsupportedErr: true, - instanceTypeName: "GPU", + amiFamily: api.NodeImageFamilyAmazonLinux2023, + gpuInstanceType: "g4dn.xlarge", }), Entry("AL2", gpuInstanceEntry{ gpuInstanceType: "g4dn.xlarge", diff --git a/pkg/apis/eksctl.io/v1alpha5/iam.go b/pkg/apis/eksctl.io/v1alpha5/iam.go index 85e93d6a38..1a446dd5e5 100644 --- a/pkg/apis/eksctl.io/v1alpha5/iam.go +++ b/pkg/apis/eksctl.io/v1alpha5/iam.go @@ -52,9 +52,8 @@ type ClusterIAM struct { // See [IAM Service Accounts](/usage/iamserviceaccounts/#usage-with-config-files) // +optional ServiceAccounts []*ClusterIAMServiceAccount `json:"serviceAccounts,omitempty"` - // pod identity associations to create in the cluster. - // See [Pod Identity Associations](TBD) + // See [Pod Identity Associations](/usage/pod-identity-associations) // +optional PodIdentityAssociations []PodIdentityAssociation `json:"podIdentityAssociations,omitempty"` diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go index 94b67afd7d..c9ceb24a7f 100644 --- a/pkg/apis/eksctl.io/v1alpha5/types.go +++ b/pkg/apis/eksctl.io/v1alpha5/types.go @@ -43,13 +43,14 @@ const ( Version1_29 = "1.29" - // Version1_30 represents Kubernetes version 1.30.x. Version1_30 = "1.30" + Version1_31 = "1.31" + // DefaultVersion (default) DefaultVersion = Version1_30 - LatestVersion = Version1_30 + LatestVersion = Version1_31 DockershimDeprecationVersion = Version1_24 ) @@ -98,8 +99,8 @@ const ( // Not yet supported versions const ( - // Version1_31 represents Kubernetes version 1.31.x - Version1_31 = "1.31" + // Version1_32 represents Kubernetes version 1.32.x + Version1_32 = "1.32" ) const ( @@ -172,6 +173,9 @@ const ( // RegionAPSouthEast4 represents the Asia-Pacific South East Region Melbourne RegionAPSouthEast4 = "ap-southeast-4" + // RegionAPSouthEast5 represents the Asia-Pacific South East Region Kuala Lumpur + RegionAPSouthEast5 = "ap-southeast-5" + // RegionAPSouth1 represents the Asia-Pacific South Region Mumbai RegionAPSouth1 = "ap-south-1" @@ -393,6 +397,10 @@ const ( // eksResourceAccountAPSouthEast4 defines the AWS EKS account ID that provides node resources in ap-southeast-4 eksResourceAccountAPSouthEast4 = "491585149902" + + // eksResourceAccountAPSouthEast5 defines the AWS EKS account ID that provides node resources in ap-southeast-5 + eksResourceAccountAPSouthEast5 = "151610086707" + // eksResourceAccountUSISOEast1 defines the AWS EKS account ID that provides node resources in us-iso-east-1 eksResourceAccountUSISOEast1 = "725322719131" @@ -443,17 +451,6 @@ const ( IPV6Family = "IPv6" ) -// Values for core addons -const ( - minimumVPCCNIVersionForIPv6 = "1.10.0" - VPCCNIAddon = "vpc-cni" - KubeProxyAddon = "kube-proxy" - CoreDNSAddon = "coredns" - PodIdentityAgentAddon = "eks-pod-identity-agent" - AWSEBSCSIDriverAddon = "aws-ebs-csi-driver" - AWSEFSCSIDriverAddon = "aws-efs-csi-driver" -) - // supported version of Karpenter const ( supportedKarpenterVersion = "v0.20.0" @@ -550,6 +547,7 @@ func SupportedRegions() []string { RegionAPSouthEast2, RegionAPSouthEast3, RegionAPSouthEast4, + RegionAPSouthEast5, RegionAPSouth1, RegionAPSouth2, RegionAPEast1, @@ -610,6 +608,7 @@ func SupportedVersions() []string { Version1_28, Version1_29, Version1_30, + Version1_31, } } @@ -696,6 +695,8 @@ func EKSResourceAccountID(region string) string { return eksResourceAccountAPSouthEast3 case RegionAPSouthEast4: return eksResourceAccountAPSouthEast4 + case RegionAPSouthEast5: + return eksResourceAccountAPSouthEast5 case RegionILCentral1: return eksResourceAccountILCentral1 case RegionUSISOEast1: @@ -995,6 +996,9 @@ type ClusterConfig struct { // Spot Ocean. // +optional SpotOcean *SpotOceanCluster `json:"spotOcean,omitempty"` + + // ZonalShiftConfig specifies the zonal shift configuration. + ZonalShiftConfig *ZonalShiftConfig `json:"zonalShiftConfig,omitempty"` } // Outpost holds the Outpost configuration. @@ -1022,6 +1026,12 @@ func (o *Outpost) HasPlacementGroup() bool { return o.ControlPlanePlacement != nil } +// ZonalShiftConfig holds the zonal shift configuration. +type ZonalShiftConfig struct { + // Enabled enables or disables zonal shift. + Enabled *bool `json:"enabled,omitempty"` +} + // OutpostInfo describes the Outpost info. type OutpostInfo interface { // IsControlPlaneOnOutposts returns true if the control plane is on Outposts. @@ -1692,6 +1702,7 @@ type ( Headroom *SpotOceanHeadroom `json:"headrooms,omitempty"` // +optional ResourceLimits *SpotOceanClusterResourceLimits `json:"resourceLimits,omitempty"` + Down *AutoScalerDown `json:"down,omitempty"` } // SpotOceanVirtualNodeGroupAutoScaler holds the auto scaler configuration used by Spot Ocean. @@ -1722,6 +1733,16 @@ type ( MaxMemoryGiB *int `json:"maxMemoryGib,omitempty"` } + AutoScalerDown struct { + EvaluationPeriods *int `json:"evaluationPeriods,omitempty"` + MaxScaleDownPercentage *float64 `json:"maxScaleDownPercentage,omitempty"` + AggressiveScaleDown *AggressiveScaleDown `json:"aggressiveScaleDown,omitempty"` + } + + AggressiveScaleDown struct { + IsEnabled *bool `json:"isEnabled,omitempty"` + } + // SpotOceanVirtualNodeGroupResourceLimits holds the resource limits configuration used by Spot Ocean. SpotOceanVirtualNodeGroupResourceLimits struct { // +optional diff --git a/pkg/apis/eksctl.io/v1alpha5/validation.go b/pkg/apis/eksctl.io/v1alpha5/validation.go index 1ac66eab99..81b03b4d50 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation.go @@ -661,12 +661,11 @@ func validateNodeGroupBase(np NodePool, path string, controlPlaneOnOutposts bool instanceType := SelectInstanceType(np) - if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 && instanceutils.IsNvidiaInstanceType(instanceType) { - return ErrUnsupportedInstanceTypes("GPU", NodeImageFamilyAmazonLinux2023, - fmt.Sprintf("EKS accelerated AMIs based on %s will be available at a later date", NodeImageFamilyAmazonLinux2023)) - } + if ng.AMIFamily != NodeImageFamilyAmazonLinux2023 && + ng.AMIFamily != NodeImageFamilyAmazonLinux2 && + ng.AMIFamily != NodeImageFamilyBottlerocket && + ng.AMIFamily != "" { - if ng.AMIFamily != NodeImageFamilyAmazonLinux2 && ng.AMIFamily != NodeImageFamilyBottlerocket && ng.AMIFamily != "" { if instanceutils.IsNvidiaInstanceType(instanceType) { logger.Warning(GPUDriversWarning(ng.AMIFamily)) } @@ -676,17 +675,35 @@ func validateNodeGroupBase(np NodePool, path string, controlPlaneOnOutposts bool } } - if ng.AMIFamily != NodeImageFamilyAmazonLinux2 && ng.AMIFamily != "" { - // Only AL2 supports Inferentia hosts. + if ng.AMIFamily != NodeImageFamilyAmazonLinux2 && + ng.AMIFamily != NodeImageFamilyAmazonLinux2023 && + ng.AMIFamily != "" { + // Only AL2 and AL2023 support Inferentia hosts. if instanceutils.IsInferentiaInstanceType(instanceType) { return ErrUnsupportedInstanceTypes("Inferentia", ng.AMIFamily, fmt.Sprintf("please use %s instead", NodeImageFamilyAmazonLinux2)) } - // Only AL2 supports Trainium hosts. + // Only AL2 and AL2023 support Trainium hosts. if instanceutils.IsTrainiumInstanceType(instanceType) { return ErrUnsupportedInstanceTypes("Trainium", ng.AMIFamily, fmt.Sprintf("please use %s instead", NodeImageFamilyAmazonLinux2)) } } + if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 { + fieldNotSupported := func(field string) error { + return &unsupportedFieldError{ + ng: ng, + path: path, + field: field, + } + } + if ng.PreBootstrapCommands != nil { + return fieldNotSupported("preBootstrapCommands") + } + if ng.OverrideBootstrapCommand != nil { + return fieldNotSupported("overrideBootstrapCommand") + } + } + if ng.CapacityReservation != nil { if ng.CapacityReservation.CapacityReservationPreference != nil { if ng.CapacityReservation.CapacityReservationTarget != nil { @@ -871,13 +888,6 @@ func ValidateNodeGroup(i int, ng *NodeGroup, cfg *ClusterConfig) error { if ng.KubeletExtraConfig != nil { return fieldNotSupported("kubeletExtraConfig") } - } else if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 { - if ng.PreBootstrapCommands != nil { - return fieldNotSupported("preBootstrapCommands") - } - if ng.OverrideBootstrapCommand != nil { - return fieldNotSupported("overrideBootstrapCommand") - } } else if ng.AMIFamily == NodeImageFamilyBottlerocket { if ng.KubeletExtraConfig != nil { return fieldNotSupported("kubeletExtraConfig") diff --git a/pkg/apis/eksctl.io/v1alpha5/validation_test.go b/pkg/apis/eksctl.io/v1alpha5/validation_test.go index a34a1880d2..59ab318a51 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation_test.go @@ -171,14 +171,6 @@ var _ = Describe("ClusterConfig validation", func() { errMsg := fmt.Sprintf("overrideBootstrapCommand is required when using a custom AMI based on %s", ng0.AMIFamily) Expect(api.ValidateNodeGroup(0, ng0, cfg)).To(MatchError(ContainSubstring(errMsg))) }) - It("should not require overrideBootstrapCommand if ami is set and type is AmazonLinux2023", func() { - cfg := api.NewClusterConfig() - ng0 := cfg.NewNodeGroup() - ng0.Name = "node-group" - ng0.AMI = "ami-1234" - ng0.AMIFamily = api.NodeImageFamilyAmazonLinux2023 - Expect(api.ValidateNodeGroup(0, ng0, cfg)).To(Succeed()) - }) It("should not require overrideBootstrapCommand if ami is set and type is Bottlerocket", func() { cfg := api.NewClusterConfig() ng0 := cfg.NewNodeGroup() @@ -204,15 +196,6 @@ var _ = Describe("ClusterConfig validation", func() { ng0.OverrideBootstrapCommand = aws.String("echo 'yo'") Expect(api.ValidateNodeGroup(0, ng0, cfg)).To(Succeed()) }) - It("should throw an error if overrideBootstrapCommand is set and type is AmazonLinux2023", func() { - cfg := api.NewClusterConfig() - ng0 := cfg.NewNodeGroup() - ng0.Name = "node-group" - ng0.AMI = "ami-1234" - ng0.AMIFamily = api.NodeImageFamilyAmazonLinux2023 - ng0.OverrideBootstrapCommand = aws.String("echo 'yo'") - Expect(api.ValidateNodeGroup(0, ng0, cfg)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) - }) It("should throw an error if overrideBootstrapCommand is set and type is Bottlerocket", func() { cfg := api.NewClusterConfig() ng0 := cfg.NewNodeGroup() @@ -2104,7 +2087,40 @@ var _ = Describe("ClusterConfig validation", func() { err := api.ValidateManagedNodeGroup(0, ng) Expect(err).To(MatchError(ContainSubstring("eksctl does not support configuring maxPodsPerNode EKS-managed nodes"))) }) - }) + It("returns an error when setting preBootstrapCommands for self-managed nodegroups", func() { + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() + ng.Name = "node-group" + ng.AMI = "ami-1234" + ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 + ng.PreBootstrapCommands = []string{"echo 'rubarb'"} + Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(fmt.Sprintf("preBootstrapCommands is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) + }) + It("returns an error when setting overrideBootstrapCommand for self-managed nodegroups", func() { + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() + ng.Name = "node-group" + ng.AMI = "ami-1234" + ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 + ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'") + Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) + }) + It("returns an error when setting preBootstrapCommands for EKS-managed nodegroups", func() { + ng := api.NewManagedNodeGroup() + ng.Name = "node-group" + ng.AMI = "ami-1234" + ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 + ng.PreBootstrapCommands = []string{"echo 'rubarb'"} + Expect(api.ValidateManagedNodeGroup(0, ng)).To(MatchError(ContainSubstring(fmt.Sprintf("preBootstrapCommands is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) + }) + It("returns an error when setting overrideBootstrapCommand for EKS-managed nodegroups", func() { + ng := api.NewManagedNodeGroup() + ng.Name = "node-group" + ng.AMI = "ami-1234" + ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 + ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'") + Expect(api.ValidateManagedNodeGroup(0, ng)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) + }) Describe("Windows node groups", func() { It("returns an error with unsupported fields", func() { diff --git a/pkg/az/az.go b/pkg/az/az.go index d2eab86619..c9bc5a8f83 100644 --- a/pkg/az/az.go +++ b/pkg/az/az.go @@ -19,7 +19,10 @@ import ( ) var zoneIDsToAvoid = map[string][]string{ - api.RegionCNNorth1: {"cnn1-az4"}, // https://github.com/eksctl-io/eksctl/issues/3916 + api.RegionCNNorth1: {"cnn1-az4"}, // https://github.com/eksctl-io/eksctl/issues/3916 + api.RegionUSEast1: {"use1-az3"}, + api.RegionUSWest1: {"usw1-az2"}, + api.RegionCACentral1: {"cac1-az3"}, } func GetAvailabilityZones(ctx context.Context, ec2API awsapi.EC2, region string, spec *api.ClusterConfig) ([]string, error) { diff --git a/pkg/az/az_test.go b/pkg/az/az_test.go index 919be08598..d6dbebaae0 100644 --- a/pkg/az/az_test.go +++ b/pkg/az/az_test.go @@ -215,7 +215,7 @@ var _ = Describe("AZ", func() { }, LocationType: ec2types.LocationTypeAvailabilityZone, MaxResults: aws.Int32(100), - }).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ NextToken: aws.String("token"), InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { @@ -249,7 +249,7 @@ var _ = Describe("AZ", func() { LocationType: ec2types.LocationTypeAvailabilityZone, MaxResults: aws.Int32(100), NextToken: aws.String("token"), - }).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { InstanceType: "t2.medium", @@ -304,6 +304,128 @@ var _ = Describe("AZ", func() { }) }) + type unsupportedZoneEntry struct { + region string + zoneNameToIDs map[string]string + expectedZones []string + } + DescribeTable("region with unsupported zone IDs", func(e unsupportedZoneEntry) { + var azs []ec2types.AvailabilityZone + for zoneName, zoneID := range e.zoneNameToIDs { + azs = append(azs, createAvailabilityZoneWithID(e.region, ec2types.AvailabilityZoneStateAvailable, zoneName, zoneID)) + } + mockProvider := mockprovider.NewMockProvider() + mockProvider.MockEC2().On("DescribeAvailabilityZones", mock.Anything, &ec2.DescribeAvailabilityZonesInput{ + Filters: []ec2types.Filter{ + { + Name: aws.String("region-name"), + Values: []string{e.region}, + }, + { + Name: aws.String("state"), + Values: []string{string(ec2types.AvailabilityZoneStateAvailable)}, + }, + { + Name: aws.String("zone-type"), + Values: []string{string(ec2types.LocationTypeAvailabilityZone)}, + }, + }, + }).Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: azs, + }, nil) + mockProvider.MockEC2().On("DescribeInstanceTypeOfferings", mock.Anything, &ec2.DescribeInstanceTypeOfferingsInput{ + Filters: []ec2types.Filter{ + { + Name: aws.String("instance-type"), + Values: []string{"t2.small", "t2.medium"}, + }, + { + Name: aws.String("location"), + Values: []string{"zone1", "zone2", "zone3", "zone4"}, + }, + }, + LocationType: ec2types.LocationTypeAvailabilityZone, + MaxResults: aws.Int32(100), + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + NextToken: aws.String("token"), + InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ + { + InstanceType: "t2.small", + Location: aws.String("zone1"), + LocationType: "availability-zone", + }, + { + InstanceType: "t2.small", + Location: aws.String("zone2"), + LocationType: "availability-zone", + }, + { + InstanceType: "t2.small", + Location: aws.String("zone4"), + LocationType: "availability-zone", + }, + { + InstanceType: "t2.small", + Location: aws.String("zone3"), + LocationType: "availability-zone", + }, + }, + }, nil) + clusterConfig := api.NewClusterConfig() + clusterConfig.Metadata.Region = e.region + clusterConfig.NodeGroups = []*api.NodeGroup{ + { + NodeGroupBase: &api.NodeGroupBase{ + Name: "test-az-1", + }, + }, + { + NodeGroupBase: &api.NodeGroupBase{ + Name: "test-az-2", + }, + }, + } + zones, err := az.GetAvailabilityZones(context.Background(), mockProvider.MockEC2(), e.region, clusterConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(zones).To(ConsistOf(e.expectedZones)) + }, + Entry(api.RegionCNNorth1, unsupportedZoneEntry{ + region: api.RegionCNNorth1, + zoneNameToIDs: map[string]string{ + "zone1": "cnn1-az1", + "zone2": "cnn1-az2", + "zone4": "cnn1-az4", + }, + expectedZones: []string{"zone1", "zone2"}, + }), + Entry(api.RegionUSEast1, unsupportedZoneEntry{ + region: api.RegionUSEast1, + zoneNameToIDs: map[string]string{ + "zone1": "use1-az1", + "zone2": "use1-az3", + "zone3": "use1-az2", + }, + expectedZones: []string{"zone1", "zone3"}, + }), + Entry(api.RegionUSWest1, unsupportedZoneEntry{ + region: api.RegionUSWest1, + zoneNameToIDs: map[string]string{ + "zone1": "usw1-az2", + "zone2": "usw1-az1", + "zone3": "usw1-az3", + }, + expectedZones: []string{"zone2", "zone3"}, + }), + Entry(api.RegionCACentral1, unsupportedZoneEntry{ + region: api.RegionCACentral1, + zoneNameToIDs: map[string]string{ + "zone1": "cac1-az1", + "zone2": "cac1-az2", + "zone3": "cac1-az3", + }, + expectedZones: []string{"zone1", "zone2"}, + }), + ) When("the region contains zones that are denylisted", func() { BeforeEach(func() { region = api.RegionCNNorth1 diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index 0eac1ba87c..54cd0c5007 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -296,11 +296,12 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe } cluster := gfneks.Cluster{ - EncryptionConfig: encryptionConfigs, - Logging: makeClusterLogging(c.spec), - Name: gfnt.NewString(c.spec.Metadata.Name), - ResourcesVpcConfig: clusterVPC, - RoleArn: serviceRoleARN, + EncryptionConfig: encryptionConfigs, + Logging: makeClusterLogging(c.spec), + Name: gfnt.NewString(c.spec.Metadata.Name), + ResourcesVpcConfig: clusterVPC, + RoleArn: serviceRoleARN, + BootstrapSelfManagedAddons: gfnt.NewBoolean(false), AccessConfig: &gfneks.Cluster_AccessConfig{ AuthenticationMode: gfnt.NewString(string(c.spec.AccessConfig.AuthenticationMode)), BootstrapClusterCreatorAdminPermissions: gfnt.NewBoolean(!api.IsDisabled(c.spec.AccessConfig.BootstrapClusterCreatorAdminPermissions)), @@ -334,6 +335,11 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe kubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(ipFamily)) } cluster.KubernetesNetworkConfig = kubernetesNetworkConfig + if c.spec.ZonalShiftConfig != nil && api.IsEnabled(c.spec.ZonalShiftConfig.Enabled) { + cluster.ZonalShiftConfig = &gfneks.Cluster_ZonalShift{ + Enabled: gfnt.NewBoolean(true), + } + } c.newResource("ControlPlane", &cluster) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index f50448a291..102cec8b5f 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -2,6 +2,7 @@ package builder_test import ( "context" + _ "embed" "encoding/json" "reflect" @@ -20,8 +21,6 @@ import ( "github.com/weaveworks/eksctl/pkg/cfn/builder" "github.com/weaveworks/eksctl/pkg/cfn/builder/fakes" "github.com/weaveworks/eksctl/pkg/testutils/mockprovider" - - _ "embed" ) var _ = Describe("Cluster Template Builder", func() { @@ -669,6 +668,12 @@ var _ = Describe("Cluster Template Builder", func() { Expect(accessConfig.BootstrapClusterCreatorAdminPermissions).To(BeFalse()) }) }) + + Context("bootstrapSelfManagedAddons in default config", func() { + It("should disable default addons", func() { + Expect(clusterTemplate.Resources["ControlPlane"].Properties.BootstrapSelfManagedAddons).To(BeFalse()) + }) + }) }) Describe("GetAllOutputs", func() { diff --git a/pkg/cfn/builder/fakes/fake_cfn_template.go b/pkg/cfn/builder/fakes/fake_cfn_template.go index 34de6e9e0e..5e554f6d7e 100644 --- a/pkg/cfn/builder/fakes/fake_cfn_template.go +++ b/pkg/cfn/builder/fakes/fake_cfn_template.go @@ -25,6 +25,7 @@ type Properties struct { Description string Tags []Tag SecurityGroupIngress []SGIngress + BootstrapSelfManagedAddons bool GroupID interface{} SourceSecurityGroupID interface{} DestinationSecurityGroupID interface{} diff --git a/pkg/cfn/builder/iam.go b/pkg/cfn/builder/iam.go index 422802c064..b534edfa46 100644 --- a/pkg/cfn/builder/iam.go +++ b/pkg/cfn/builder/iam.go @@ -217,12 +217,6 @@ func NewIAMRoleResourceSetForServiceAccount(spec *api.ClusterIAMServiceAccount, } } -func NewIAMRoleResourceSetForPodIdentityWithTrustStatements(spec *api.PodIdentityAssociation, trustStatements []api.IAMStatement) *IAMRoleResourceSet { - rs := NewIAMRoleResourceSetForPodIdentity(spec) - rs.trustStatements = trustStatements - return rs -} - func NewIAMRoleResourceSetForPodIdentity(spec *api.PodIdentityAssociation) *IAMRoleResourceSet { return &IAMRoleResourceSet{ template: cft.NewTemplate(), diff --git a/pkg/cfn/builder/iam_test.go b/pkg/cfn/builder/iam_test.go index d87303cd80..0d793850e5 100644 --- a/pkg/cfn/builder/iam_test.go +++ b/pkg/cfn/builder/iam_test.go @@ -295,6 +295,7 @@ var _ = Describe("template builder for IAM", func() { } ]`)) Expect(t).To(HaveOutputWithValue(outputs.IAMServiceAccountRoleName, `{ "Fn::GetAtt": "Role1.Arn" }`)) + Expect(t).To(HaveResourceWithPropertyValue("PolicyAWSLoadBalancerController", "PolicyDocument", expectedAWSLoadBalancerControllerPolicyDocument)) Expect(t).To(HaveResourceWithPropertyValue("PolicyEBSCSIController", "PolicyDocument", expectedEbsPolicyDocument)) }) @@ -467,6 +468,276 @@ const expectedAssumeRolePolicyDocument = `{ "Version": "2012-10-17" }` +const expectedAWSLoadBalancerControllerPolicyDocument = `{ + "Statement": [ + { + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + }, + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:ec2:*:*:security-group/*" + } + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:ec2:*:*:security-group/*" + } + }, + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + }, + "Effect": "Allow", + "Resource": [ + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:loadbalancer/net/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:loadbalancer/app/*/*" + } + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:listener/net/*/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:listener/app/*/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:listener-rule/net/*/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + } + ] + }, + { + "Action": [ + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup" + ], + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + }, + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + } + }, + "Effect": "Allow", + "Resource": [ + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:loadbalancer/net/*/*" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:loadbalancer/app/*/*" + } + ] + }, + { + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:elasticloadbalancing:*:*:targetgroup/*/*" + } + }, + { + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +}` + const expectedEbsPolicyDocument = `{ "Statement": [ { diff --git a/pkg/cfn/builder/karpenter.go b/pkg/cfn/builder/karpenter.go index dc1ac5c3fb..5a42218aba 100644 --- a/pkg/cfn/builder/karpenter.go +++ b/pkg/cfn/builder/karpenter.go @@ -49,9 +49,15 @@ const ( ec2DescribeImages = "ec2:DescribeImages" ec2DescribeSpotPriceHistory = "ec2:DescribeSpotPriceHistory" // IAM - iamPassRole = "iam:PassRole" - iamCreateServiceLinkedRole = "iam:CreateServiceLinkedRole" - ssmGetParameter = "ssm:GetParameter" + iamPassRole = "iam:PassRole" + iamCreateServiceLinkedRole = "iam:CreateServiceLinkedRole" + iamGetInstanceProfile = "iam:GetInstanceProfile" + iamCreateInstanceProfile = "iam:CreateInstanceProfile" + iamDeleteInstanceProfile = "iam:DeleteInstanceProfile" + iamTagInstanceProfile = "iam:TagInstanceProfile" + iamAddRoleToInstanceProfile = "iam:AddRoleToInstanceProfile" + // SSM + ssmGetParameter = "ssm:GetParameter" // Pricing pricingGetProducts = "pricing:GetProducts" // SQS @@ -165,6 +171,11 @@ func (k *KarpenterResourceSet) addResourcesForKarpenter() error { ec2DescribeSpotPriceHistory, iamPassRole, iamCreateServiceLinkedRole, + iamGetInstanceProfile, + iamCreateInstanceProfile, + iamDeleteInstanceProfile, + iamTagInstanceProfile, + iamAddRoleToInstanceProfile, ssmGetParameter, pricingGetProducts, }, diff --git a/pkg/cfn/builder/karpenter_test.go b/pkg/cfn/builder/karpenter_test.go index 11935ea3a1..39605cd7ce 100644 --- a/pkg/cfn/builder/karpenter_test.go +++ b/pkg/cfn/builder/karpenter_test.go @@ -125,6 +125,11 @@ var expectedTemplate = `{ "ec2:DescribeSpotPriceHistory", "iam:PassRole", "iam:CreateServiceLinkedRole", + "iam:GetInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:TagInstanceProfile", + "iam:AddRoleToInstanceProfile", "ssm:GetParameter", "pricing:GetProducts" ], @@ -262,6 +267,11 @@ var expectedTemplateWithPermissionBoundary = `{ "ec2:DescribeSpotPriceHistory", "iam:PassRole", "iam:CreateServiceLinkedRole", + "iam:GetInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:TagInstanceProfile", + "iam:AddRoleToInstanceProfile", "ssm:GetParameter", "pricing:GetProducts" ], @@ -424,6 +434,11 @@ var expectedTemplateWithSpotInterruptionQueue = `{ "ec2:DescribeSpotPriceHistory", "iam:PassRole", "iam:CreateServiceLinkedRole", + "iam:GetInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:TagInstanceProfile", + "iam:AddRoleToInstanceProfile", "ssm:GetParameter", "pricing:GetProducts" ], diff --git a/pkg/cfn/builder/managed_nodegroup.go b/pkg/cfn/builder/managed_nodegroup.go index b90d07eb44..18cf5c4193 100644 --- a/pkg/cfn/builder/managed_nodegroup.go +++ b/pkg/cfn/builder/managed_nodegroup.go @@ -263,41 +263,45 @@ func validateLaunchTemplate(launchTemplateData *ec2types.ResponseLaunchTemplateD func getAMIType(ng *api.ManagedNodeGroup, instanceType string) ekstypes.AMITypes { amiTypeMapping := map[string]struct { - X86x64 ekstypes.AMITypes - X86GPU ekstypes.AMITypes - ARM ekstypes.AMITypes - ARMGPU ekstypes.AMITypes + X86x64 ekstypes.AMITypes + X86Nvidia ekstypes.AMITypes + X86Neuron ekstypes.AMITypes + ARM ekstypes.AMITypes + ARMGPU ekstypes.AMITypes }{ api.NodeImageFamilyAmazonLinux2023: { - X86x64: ekstypes.AMITypesAl2023X8664Standard, - ARM: ekstypes.AMITypesAl2023Arm64Standard, + X86x64: ekstypes.AMITypesAl2023X8664Standard, + X86Nvidia: ekstypes.AMITypesAl2023X8664Nvidia, + X86Neuron: ekstypes.AMITypesAl2023X8664Neuron, + ARM: ekstypes.AMITypesAl2023Arm64Standard, }, api.NodeImageFamilyAmazonLinux2: { - X86x64: ekstypes.AMITypesAl2X8664, - X86GPU: ekstypes.AMITypesAl2X8664Gpu, - ARM: ekstypes.AMITypesAl2Arm64, + X86x64: ekstypes.AMITypesAl2X8664, + X86Nvidia: ekstypes.AMITypesAl2X8664Gpu, + X86Neuron: ekstypes.AMITypesAl2X8664Gpu, + ARM: ekstypes.AMITypesAl2Arm64, }, api.NodeImageFamilyBottlerocket: { - X86x64: ekstypes.AMITypesBottlerocketX8664, - X86GPU: ekstypes.AMITypesBottlerocketX8664Nvidia, - ARM: ekstypes.AMITypesBottlerocketArm64, - ARMGPU: ekstypes.AMITypesBottlerocketArm64Nvidia, + X86x64: ekstypes.AMITypesBottlerocketX8664, + X86Nvidia: ekstypes.AMITypesBottlerocketX8664Nvidia, + ARM: ekstypes.AMITypesBottlerocketArm64, + ARMGPU: ekstypes.AMITypesBottlerocketArm64Nvidia, }, api.NodeImageFamilyWindowsServer2019FullContainer: { - X86x64: ekstypes.AMITypesWindowsFull2019X8664, - X86GPU: ekstypes.AMITypesWindowsFull2019X8664, + X86x64: ekstypes.AMITypesWindowsFull2019X8664, + X86Nvidia: ekstypes.AMITypesWindowsFull2019X8664, }, api.NodeImageFamilyWindowsServer2019CoreContainer: { - X86x64: ekstypes.AMITypesWindowsCore2019X8664, - X86GPU: ekstypes.AMITypesWindowsCore2019X8664, + X86x64: ekstypes.AMITypesWindowsCore2019X8664, + X86Nvidia: ekstypes.AMITypesWindowsCore2019X8664, }, api.NodeImageFamilyWindowsServer2022FullContainer: { - X86x64: ekstypes.AMITypesWindowsFull2022X8664, - X86GPU: ekstypes.AMITypesWindowsFull2022X8664, + X86x64: ekstypes.AMITypesWindowsFull2022X8664, + X86Nvidia: ekstypes.AMITypesWindowsFull2022X8664, }, api.NodeImageFamilyWindowsServer2022CoreContainer: { - X86x64: ekstypes.AMITypesWindowsCore2022X8664, - X86GPU: ekstypes.AMITypesWindowsCore2022X8664, + X86x64: ekstypes.AMITypesWindowsCore2022X8664, + X86Nvidia: ekstypes.AMITypesWindowsCore2022X8664, }, } @@ -307,13 +311,14 @@ func getAMIType(ng *api.ManagedNodeGroup, instanceType string) ekstypes.AMITypes } switch { - case instanceutils.IsGPUInstanceType(instanceType): - if instanceutils.IsARMInstanceType(instanceType) { - return amiType.ARMGPU - } - return amiType.X86GPU + case instanceutils.IsARMGPUInstanceType(instanceType): + return amiType.ARMGPU case instanceutils.IsARMInstanceType(instanceType): return amiType.ARM + case instanceutils.IsNvidiaInstanceType(instanceType): + return amiType.X86Nvidia + case instanceutils.IsNeuronInstanceType(instanceType): + return amiType.X86Neuron default: return amiType.X86x64 } diff --git a/pkg/cfn/builder/managed_nodegroup_ami_type_test.go b/pkg/cfn/builder/managed_nodegroup_ami_type_test.go index 2f3772b1e5..3839b44939 100644 --- a/pkg/cfn/builder/managed_nodegroup_ami_type_test.go +++ b/pkg/cfn/builder/managed_nodegroup_ami_type_test.go @@ -77,23 +77,24 @@ var _ = DescribeTable("Managed Nodegroup AMI type", func(e amiTypeEntry) { expectedAMIType: "AL2_x86_64", }), - Entry("AMI type", amiTypeEntry{ + Entry("default Nvidia GPU instance type", amiTypeEntry{ nodeGroup: &api.ManagedNodeGroup{ NodeGroupBase: &api.NodeGroupBase{ - Name: "test", + Name: "test", + InstanceType: "p2.xlarge", }, }, - expectedAMIType: "AL2023_x86_64_STANDARD", + expectedAMIType: "AL2023_x86_64_NVIDIA", }), - Entry("default GPU instance type", amiTypeEntry{ + Entry("default Neuron GPU instance type", amiTypeEntry{ nodeGroup: &api.ManagedNodeGroup{ NodeGroupBase: &api.NodeGroupBase{ Name: "test", - InstanceType: "p2.xlarge", + InstanceType: "inf1.2xlarge", }, }, - expectedAMIType: "AL2_x86_64_GPU", + expectedAMIType: "AL2023_x86_64_NEURON", }), Entry("AL2 GPU instance type", amiTypeEntry{ @@ -107,6 +108,16 @@ var _ = DescribeTable("Managed Nodegroup AMI type", func(e amiTypeEntry) { expectedAMIType: "AL2_x86_64_GPU", }), + Entry("default ARM instance type", amiTypeEntry{ + nodeGroup: &api.ManagedNodeGroup{ + NodeGroupBase: &api.NodeGroupBase{ + Name: "test", + InstanceType: "a1.2xlarge", + }, + }, + expectedAMIType: "AL2023_ARM_64_STANDARD", + }), + Entry("AL2 ARM instance type", amiTypeEntry{ nodeGroup: &api.ManagedNodeGroup{ NodeGroupBase: &api.NodeGroupBase{ diff --git a/pkg/cfn/builder/nodegroup.go b/pkg/cfn/builder/nodegroup.go index 2a8cbd5c24..8dc4ff8fa5 100644 --- a/pkg/cfn/builder/nodegroup.go +++ b/pkg/cfn/builder/nodegroup.go @@ -940,6 +940,17 @@ func (n *NodeGroupResourceSet) newNodeGroupSpotOceanClusterResource(launchTempla MaxMemoryGiB: l.MaxMemoryGiB, } } + if d := autoScaler.Down; d != nil { + cluster.AutoScaler.Down = &spot.AutoScalerDown{ + EvaluationPeriods: d.EvaluationPeriods, + MaxScaleDownPercentage: d.MaxScaleDownPercentage, + } + if d.AggressiveScaleDown != nil { + cluster.AutoScaler.Down.AggressiveScaleDown = &spot.AggressiveScaleDown{ + IsEnabled: d.AggressiveScaleDown.IsEnabled, + } + } + } } } } diff --git a/pkg/cfn/builder/nodegroup_subnets_test.go b/pkg/cfn/builder/nodegroup_subnets_test.go index 8eeb3c19a7..75b13a1a6a 100644 --- a/pkg/cfn/builder/nodegroup_subnets_test.go +++ b/pkg/cfn/builder/nodegroup_subnets_test.go @@ -267,7 +267,7 @@ var _ = Describe("AssignSubnets", func() { } }, updateEC2Mocks: func(e *mocksv2.EC2) { - e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything). + e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything, mock.Anything). Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { @@ -321,7 +321,7 @@ var _ = Describe("AssignSubnets", func() { } }, updateEC2Mocks: func(e *mocksv2.EC2) { - e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything). + e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything, mock.Anything). Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { @@ -375,7 +375,7 @@ var _ = Describe("AssignSubnets", func() { } }, updateEC2Mocks: func(e *mocksv2.EC2) { - e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything). + e.On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything, mock.Anything). Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { @@ -472,7 +472,7 @@ func mockDescribeSubnets(ec2Mock *mocksv2.EC2, zoneName, vpcID string) { } func mockDescribeSubnetsWithOutpost(ec2Mock *mocksv2.EC2, zoneName, vpcID string, outpostARN *string) { - ec2Mock.On("DescribeSubnets", mock.Anything, mock.Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { + ec2Mock.On("DescribeSubnets", mock.Anything, mock.Anything, mock.Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { return &ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { @@ -559,7 +559,7 @@ func mockSubnetsAndAZInstanceSupport( AvailabilityZones: azs, }, nil) provider.MockEC2(). - On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything). + On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything, mock.Anything). Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: offerings, }, nil) diff --git a/pkg/cfn/builder/statement.go b/pkg/cfn/builder/statement.go index da2789a5a2..ad01b49c83 100644 --- a/pkg/cfn/builder/statement.go +++ b/pkg/cfn/builder/statement.go @@ -42,6 +42,7 @@ func loadBalancerControllerStatements() []cft.MapOfInterfaces { "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerAttributes", "elasticloadbalancing:DescribeListenerCertificates", "elasticloadbalancing:DescribeSSLPolicies", "elasticloadbalancing:DescribeRules", @@ -190,6 +191,7 @@ func loadBalancerControllerStatements() []cft.MapOfInterfaces { { "Effect": effectAllow, "Action": []string{ + "elasticloadbalancing:ModifyListenerAttributes", "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:SetIpAddressType", "elasticloadbalancing:SetSecurityGroups", diff --git a/pkg/cfn/builder/vpc_endpoint_test.go b/pkg/cfn/builder/vpc_endpoint_test.go index 24b1ee4e78..0292d2c2ec 100644 --- a/pkg/cfn/builder/vpc_endpoint_test.go +++ b/pkg/cfn/builder/vpc_endpoint_test.go @@ -281,7 +281,7 @@ var _ = Describe("VPC Endpoint Builder", func() { } provider.MockEC2().On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(output, nil) + }), mock.Anything).Return(output, nil) return provider }, err: "subnets must be associated with a non-main route table", @@ -440,7 +440,7 @@ func mockDescribeRouteTables(provider *mockprovider.MockProvider, subnetIDs []st provider.MockEC2().On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(output, nil) + }), mock.Anything).Return(output, nil) } func mockDescribeRouteTablesSame(provider *mockprovider.MockProvider, subnetIDs []string) { @@ -466,7 +466,15 @@ func mockDescribeRouteTablesSame(provider *mockprovider.MockProvider, subnetIDs provider.MockEC2().On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(output, nil) + }), mock.Anything).Return(output, nil) +} + +func makeZones(region string, count int) []string { + var ret []string + for i := 0; i < count; i++ { + ret = append(ret, fmt.Sprintf("%s%c", region, 'a'+i)) + } + return ret } func makeZones(region string, count int) []string { diff --git a/pkg/cfn/builder/vpc_existing_test.go b/pkg/cfn/builder/vpc_existing_test.go index 39978f1b50..180bf4cbc8 100644 --- a/pkg/cfn/builder/vpc_existing_test.go +++ b/pkg/cfn/builder/vpc_existing_test.go @@ -222,7 +222,7 @@ var _ = Describe("Existing VPC", func() { mockEC2.On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(mockResultFn, nil) + }), mock.Anything).Return(mockResultFn, nil) }) It("the private subnet resource values are loaded into the VPCResource with route table association", func() { @@ -245,7 +245,7 @@ var _ = Describe("Existing VPC", func() { rtOutput.RouteTables[0].Associations[0].Main = aws.Bool(true) mockEC2.On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(rtOutput, nil) + }), mock.Anything).Return(rtOutput, nil) }) It("returns an error", func() { @@ -258,7 +258,7 @@ var _ = Describe("Existing VPC", func() { rtOutput.RouteTables[0].Associations[0].SubnetId = aws.String("fake") mockEC2.On("DescribeRouteTables", mock.Anything, mock.MatchedBy(func(input *ec2.DescribeRouteTablesInput) bool { return len(input.Filters) > 0 - })).Return(rtOutput, nil) + }), mock.Anything).Return(rtOutput, nil) }) It("returns an error", func() { diff --git a/pkg/cfn/manager/api_test.go b/pkg/cfn/manager/api_test.go index cf650271a3..d6e3b6d574 100644 --- a/pkg/cfn/manager/api_test.go +++ b/pkg/cfn/manager/api_test.go @@ -403,7 +403,7 @@ var _ = Describe("StackCollection", func() { }) It("can retrieve stacks", func() { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cfn.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cfn.ListStacksOutput{ StackSummaries: []types.StackSummary{ { StackName: &stackNameWithEksctl, @@ -418,7 +418,7 @@ var _ = Describe("StackCollection", func() { When("the config stack doesn't match", func() { It("returns no stack", func() { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cfn.ListStacksOutput{}, nil) + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cfn.ListStacksOutput{}, nil) cfg.Metadata.Name = "not-this" sm := NewStackCollection(p, cfg) stack, err := sm.GetClusterStackIfExists(context.Background()) @@ -429,7 +429,7 @@ var _ = Describe("StackCollection", func() { When("ListStacks errors", func() { It("errors", func() { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(nil, errors.New("nope")) + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("nope")) sm := NewStackCollection(p, cfg) _, err := sm.GetClusterStackIfExists(context.Background()) Expect(err).To(MatchError(ContainSubstring("nope"))) diff --git a/pkg/cfn/manager/create_tasks.go b/pkg/cfn/manager/create_tasks.go index 9380d7b70f..93e18c2edb 100644 --- a/pkg/cfn/manager/create_tasks.go +++ b/pkg/cfn/manager/create_tasks.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/pkg/errors" "github.com/weaveworks/eksctl/pkg/spot" "github.com/kris-nova/logger" @@ -26,7 +25,8 @@ import ( // NewTasksToCreateCluster defines all tasks required to create a cluster along // with some nodegroups; see CreateAllNodeGroups for how onlyNodeGroupSubset works. func (c *StackCollection) NewTasksToCreateCluster(ctx context.Context, nodeGroups []*api.NodeGroup, - managedNodeGroups []*api.ManagedNodeGroup, accessConfig *api.AccessConfig, accessEntryCreator accessentry.CreatorInterface, postClusterCreationTasks ...tasks.Task) (*tasks.TaskTree, error) { + managedNodeGroups []*api.ManagedNodeGroup, accessConfig *api.AccessConfig, accessEntryCreator accessentry.CreatorInterface, nodeGroupParallelism int, postClusterCreationTasks ...tasks.Task) *tasks.TaskTree { + taskTree := tasks.TaskTree{Parallel: false} taskTree.Append(&createClusterTask{ @@ -40,29 +40,23 @@ func (c *StackCollection) NewTasksToCreateCluster(ctx context.Context, nodeGroup taskTree.Append(accessEntryCreator.CreateTasks(ctx, accessConfig.AccessEntries)) } - if len(accessConfig.AccessEntries) > 0 { - taskTree.Append(accessEntryCreator.CreateTasks(ctx, accessConfig.AccessEntries)) - } - - appendNodeGroupTasksTo := func(taskTree *tasks.TaskTree) error { + appendNodeGroupTasksTo := func(taskTree *tasks.TaskTree) { vpcImporter := vpc.NewStackConfigImporter(c.MakeClusterStackName()) - nodeGroupTasks := &tasks.TaskTree{ Parallel: true, IsSubTask: true, } - disableAccessEntryCreation := accessConfig.AuthenticationMode == ekstypes.AuthenticationModeConfigMap if oceanManagedNodeGroupTasks, err := c.NewSpotOceanNodeGroupTask(ctx, vpcImporter); oceanManagedNodeGroupTasks.Len() > 0 && err == nil { oceanManagedNodeGroupTasks.IsSubTask = true nodeGroupTasks.Parallel = false nodeGroupTasks.Append(oceanManagedNodeGroupTasks) } - if unmanagedNodeGroupTasks := c.NewUnmanagedNodeGroupTask(ctx, nodeGroups, false, false, disableAccessEntryCreation, vpcImporter); unmanagedNodeGroupTasks.Len() > 0 { + if unmanagedNodeGroupTasks := c.NewUnmanagedNodeGroupTask(ctx, nodeGroups, false, false, disableAccessEntryCreation, vpcImporter, nodeGroupParallelism); unmanagedNodeGroupTasks.Len() > 0 { unmanagedNodeGroupTasks.IsSubTask = true nodeGroupTasks.Append(unmanagedNodeGroupTasks) } - if managedNodeGroupTasks := c.NewManagedNodeGroupTask(ctx, managedNodeGroups, false, vpcImporter); managedNodeGroupTasks.Len() > 0 { + if managedNodeGroupTasks := c.NewManagedNodeGroupTask(ctx, managedNodeGroups, false, vpcImporter, nodeGroupParallelism); managedNodeGroupTasks.Len() > 0 { managedNodeGroupTasks.IsSubTask = true nodeGroupTasks.Append(managedNodeGroupTasks) } @@ -70,25 +64,20 @@ func (c *StackCollection) NewTasksToCreateCluster(ctx context.Context, nodeGroup if nodeGroupTasks.Len() > 0 { taskTree.Append(nodeGroupTasks) } - - return nil } - var appendErr error - if len(postClusterCreationTasks) > 0 { postClusterCreationTaskTree := &tasks.TaskTree{ Parallel: false, IsSubTask: true, } postClusterCreationTaskTree.Append(postClusterCreationTasks...) - appendErr = appendNodeGroupTasksTo(postClusterCreationTaskTree) + appendNodeGroupTasksTo(postClusterCreationTaskTree) taskTree.Append(postClusterCreationTaskTree) } else { - appendErr = appendNodeGroupTasksTo(&taskTree) + appendNodeGroupTasksTo(&taskTree) } - - return &taskTree, appendErr + return &taskTree } // NewSpotOceanNodeGroupTask defines tasks required to create Ocean Cluster. @@ -126,7 +115,7 @@ func (c *StackCollection) NewSpotOceanNodeGroupTask(ctx context.Context, vpcImpo } // NewUnmanagedNodeGroupTask returns tasks for creating self-managed nodegroups. -func (c *StackCollection) NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, vpcImporter vpc.Importer) *tasks.TaskTree { +func (c *StackCollection) NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, vpcImporter vpc.Importer, parallelism int) *tasks.TaskTree { task := &UnmanagedNodeGroupTask{ ClusterConfig: c.spec, NodeGroups: nodeGroups, @@ -144,12 +133,13 @@ func (c *StackCollection) NewUnmanagedNodeGroupTask(ctx context.Context, nodeGro SkipEgressRules: skipEgressRules, DisableAccessEntryCreation: disableAccessEntryCreation, VPCImporter: vpcImporter, + Parallelism: parallelism, }) } // NewManagedNodeGroupTask defines tasks required to create managed nodegroups -func (c *StackCollection) NewManagedNodeGroupTask(ctx context.Context, nodeGroups []*api.ManagedNodeGroup, forceAddCNIPolicy bool, vpcImporter vpc.Importer) *tasks.TaskTree { - taskTree := &tasks.TaskTree{Parallel: true} +func (c *StackCollection) NewManagedNodeGroupTask(ctx context.Context, nodeGroups []*api.ManagedNodeGroup, forceAddCNIPolicy bool, vpcImporter vpc.Importer, nodeGroupParallelism int) *tasks.TaskTree { + taskTree := &tasks.TaskTree{Parallel: true, Limit: nodeGroupParallelism} for _, ng := range nodeGroups { // Disable parallelisation if any tags propagation is done // since nodegroup must be created to propagate tags to its ASGs. @@ -213,7 +203,7 @@ func (c *StackCollection) NewTasksToCreateIAMServiceAccounts(serviceAccounts []* objectMeta.SetAnnotations(sa.AsObjectMeta().Annotations) objectMeta.SetLabels(sa.AsObjectMeta().Labels) if err := kubernetes.MaybeCreateServiceAccountOrUpdateMetadata(clientSet, objectMeta); err != nil { - return errors.Wrapf(err, "failed to create service account %s/%s", objectMeta.GetNamespace(), objectMeta.GetName()) + return fmt.Errorf("failed to create service account %s/%s: %w", objectMeta.GetNamespace(), objectMeta.GetName(), err) } return nil }, diff --git a/pkg/cfn/manager/fakes/fake_stack_manager.go b/pkg/cfn/manager/fakes/fake_stack_manager.go index 213fb8414a..f9316bc692 100644 --- a/pkg/cfn/manager/fakes/fake_stack_manager.go +++ b/pkg/cfn/manager/fakes/fake_stack_manager.go @@ -634,13 +634,14 @@ type FakeStackManager struct { mustUpdateStackReturnsOnCall map[int]struct { result1 error } - NewManagedNodeGroupTaskStub func(context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer) *tasks.TaskTree + NewManagedNodeGroupTaskStub func(context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer, int) *tasks.TaskTree newManagedNodeGroupTaskMutex sync.RWMutex newManagedNodeGroupTaskArgsForCall []struct { arg1 context.Context arg2 []*v1alpha5.ManagedNodeGroup arg3 bool arg4 vpc.Importer + arg5 int } newManagedNodeGroupTaskReturns struct { result1 *tasks.TaskTree @@ -663,7 +664,7 @@ type FakeStackManager struct { newTaskToDeleteUnownedNodeGroupReturnsOnCall map[int]struct { result1 tasks.Task } - NewTasksToCreateClusterStub func(context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, ...tasks.Task) (*tasks.TaskTree, error) + NewTasksToCreateClusterStub func(context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, int, ...tasks.Task) *tasks.TaskTree newTasksToCreateClusterMutex sync.RWMutex newTasksToCreateClusterArgsForCall []struct { arg1 context.Context @@ -671,7 +672,8 @@ type FakeStackManager struct { arg3 []*v1alpha5.ManagedNodeGroup arg4 *v1alpha5.AccessConfig arg5 accessentry.CreatorInterface - arg6 []tasks.Task + arg6 int + arg7 []tasks.Task } newTasksToCreateClusterReturns struct { result1 *tasks.TaskTree @@ -767,7 +769,7 @@ type FakeStackManager struct { result1 *tasks.TaskTree result2 error } - NewUnmanagedNodeGroupTaskStub func(context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer) *tasks.TaskTree + NewUnmanagedNodeGroupTaskStub func(context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer, int) *tasks.TaskTree newUnmanagedNodeGroupTaskMutex sync.RWMutex newUnmanagedNodeGroupTaskArgsForCall []struct { arg1 context.Context @@ -776,6 +778,7 @@ type FakeStackManager struct { arg4 bool arg5 bool arg6 vpc.Importer + arg7 int } newUnmanagedNodeGroupTaskReturns struct { result1 *tasks.TaskTree @@ -3802,7 +3805,7 @@ func (fake *FakeStackManager) MustUpdateStackReturnsOnCall(i int, result1 error) }{result1} } -func (fake *FakeStackManager) NewManagedNodeGroupTask(arg1 context.Context, arg2 []*v1alpha5.ManagedNodeGroup, arg3 bool, arg4 vpc.Importer) *tasks.TaskTree { +func (fake *FakeStackManager) NewManagedNodeGroupTask(arg1 context.Context, arg2 []*v1alpha5.ManagedNodeGroup, arg3 bool, arg4 vpc.Importer, arg5 int) *tasks.TaskTree { var arg2Copy []*v1alpha5.ManagedNodeGroup if arg2 != nil { arg2Copy = make([]*v1alpha5.ManagedNodeGroup, len(arg2)) @@ -3815,13 +3818,14 @@ func (fake *FakeStackManager) NewManagedNodeGroupTask(arg1 context.Context, arg2 arg2 []*v1alpha5.ManagedNodeGroup arg3 bool arg4 vpc.Importer - }{arg1, arg2Copy, arg3, arg4}) + arg5 int + }{arg1, arg2Copy, arg3, arg4, arg5}) stub := fake.NewManagedNodeGroupTaskStub fakeReturns := fake.newManagedNodeGroupTaskReturns - fake.recordInvocation("NewManagedNodeGroupTask", []interface{}{arg1, arg2Copy, arg3, arg4}) + fake.recordInvocation("NewManagedNodeGroupTask", []interface{}{arg1, arg2Copy, arg3, arg4, arg5}) fake.newManagedNodeGroupTaskMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4) + return stub(arg1, arg2, arg3, arg4, arg5) } if specificReturn { return ret.result1 @@ -3835,17 +3839,17 @@ func (fake *FakeStackManager) NewManagedNodeGroupTaskCallCount() int { return len(fake.newManagedNodeGroupTaskArgsForCall) } -func (fake *FakeStackManager) NewManagedNodeGroupTaskCalls(stub func(context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer) *tasks.TaskTree) { +func (fake *FakeStackManager) NewManagedNodeGroupTaskCalls(stub func(context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer, int) *tasks.TaskTree) { fake.newManagedNodeGroupTaskMutex.Lock() defer fake.newManagedNodeGroupTaskMutex.Unlock() fake.NewManagedNodeGroupTaskStub = stub } -func (fake *FakeStackManager) NewManagedNodeGroupTaskArgsForCall(i int) (context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer) { +func (fake *FakeStackManager) NewManagedNodeGroupTaskArgsForCall(i int) (context.Context, []*v1alpha5.ManagedNodeGroup, bool, vpc.Importer, int) { fake.newManagedNodeGroupTaskMutex.RLock() defer fake.newManagedNodeGroupTaskMutex.RUnlock() argsForCall := fake.newManagedNodeGroupTaskArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } func (fake *FakeStackManager) NewManagedNodeGroupTaskReturns(result1 *tasks.TaskTree) { @@ -3936,7 +3940,7 @@ func (fake *FakeStackManager) NewTaskToDeleteUnownedNodeGroupReturnsOnCall(i int }{result1} } -func (fake *FakeStackManager) NewTasksToCreateCluster(arg1 context.Context, arg2 []*v1alpha5.NodeGroup, arg3 []*v1alpha5.ManagedNodeGroup, arg4 *v1alpha5.AccessConfig, arg5 accessentry.CreatorInterface, arg6 ...tasks.Task) (*tasks.TaskTree, error) { +func (fake *FakeStackManager) NewTasksToCreateCluster(arg1 context.Context, arg2 []*v1alpha5.NodeGroup, arg3 []*v1alpha5.ManagedNodeGroup, arg4 *v1alpha5.AccessConfig, arg5 accessentry.CreatorInterface, arg6 int, arg7 ...tasks.Task) *tasks.TaskTree { var arg2Copy []*v1alpha5.NodeGroup if arg2 != nil { arg2Copy = make([]*v1alpha5.NodeGroup, len(arg2)) @@ -3955,14 +3959,15 @@ func (fake *FakeStackManager) NewTasksToCreateCluster(arg1 context.Context, arg2 arg3 []*v1alpha5.ManagedNodeGroup arg4 *v1alpha5.AccessConfig arg5 accessentry.CreatorInterface - arg6 []tasks.Task - }{arg1, arg2Copy, arg3Copy, arg4, arg5, arg6}) + arg6 int + arg7 []tasks.Task + }{arg1, arg2Copy, arg3Copy, arg4, arg5, arg6, arg7}) stub := fake.NewTasksToCreateClusterStub fakeReturns := fake.newTasksToCreateClusterReturns - fake.recordInvocation("NewTasksToCreateCluster", []interface{}{arg1, arg2Copy, arg3Copy, arg4, arg5, arg6}) + fake.recordInvocation("NewTasksToCreateCluster", []interface{}{arg1, arg2Copy, arg3Copy, arg4, arg5, arg6, arg7}) fake.newTasksToCreateClusterMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4, arg5, arg6...) + return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7...) } if specificReturn { return ret.result1, ret.result2 @@ -3976,20 +3981,20 @@ func (fake *FakeStackManager) NewTasksToCreateClusterCallCount() int { return len(fake.newTasksToCreateClusterArgsForCall) } -func (fake *FakeStackManager) NewTasksToCreateClusterCalls(stub func(context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, ...tasks.Task) (*tasks.TaskTree, error)) { +func (fake *FakeStackManager) NewTasksToCreateClusterCalls(stub func(context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, int, ...tasks.Task) *tasks.TaskTree) { fake.newTasksToCreateClusterMutex.Lock() defer fake.newTasksToCreateClusterMutex.Unlock() fake.NewTasksToCreateClusterStub = stub } -func (fake *FakeStackManager) NewTasksToCreateClusterArgsForCall(i int) (context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, []tasks.Task) { +func (fake *FakeStackManager) NewTasksToCreateClusterArgsForCall(i int) (context.Context, []*v1alpha5.NodeGroup, []*v1alpha5.ManagedNodeGroup, *v1alpha5.AccessConfig, accessentry.CreatorInterface, int, []tasks.Task) { fake.newTasksToCreateClusterMutex.RLock() defer fake.newTasksToCreateClusterMutex.RUnlock() argsForCall := fake.newTasksToCreateClusterArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeStackManager) NewTasksToCreateClusterReturns(result1 *tasks.TaskTree, result2 error) { +func (fake *FakeStackManager) NewTasksToCreateClusterReturns(result1 *tasks.TaskTree) { fake.newTasksToCreateClusterMutex.Lock() defer fake.newTasksToCreateClusterMutex.Unlock() fake.NewTasksToCreateClusterStub = nil @@ -3999,7 +4004,7 @@ func (fake *FakeStackManager) NewTasksToCreateClusterReturns(result1 *tasks.Task }{result1, result2} } -func (fake *FakeStackManager) NewTasksToCreateClusterReturnsOnCall(i int, result1 *tasks.TaskTree, result2 error) { +func (fake *FakeStackManager) NewTasksToCreateClusterReturnsOnCall(i int, result1 *tasks.TaskTree) { fake.newTasksToCreateClusterMutex.Lock() defer fake.newTasksToCreateClusterMutex.Unlock() fake.NewTasksToCreateClusterStub = nil @@ -4375,7 +4380,7 @@ func (fake *FakeStackManager) NewTasksToDeleteOIDCProviderWithIAMServiceAccounts }{result1, result2} } -func (fake *FakeStackManager) NewUnmanagedNodeGroupTask(arg1 context.Context, arg2 []*v1alpha5.NodeGroup, arg3 bool, arg4 bool, arg5 bool, arg6 vpc.Importer) *tasks.TaskTree { +func (fake *FakeStackManager) NewUnmanagedNodeGroupTask(arg1 context.Context, arg2 []*v1alpha5.NodeGroup, arg3 bool, arg4 bool, arg5 bool, arg6 vpc.Importer, arg7 int) *tasks.TaskTree { var arg2Copy []*v1alpha5.NodeGroup if arg2 != nil { arg2Copy = make([]*v1alpha5.NodeGroup, len(arg2)) @@ -4390,13 +4395,14 @@ func (fake *FakeStackManager) NewUnmanagedNodeGroupTask(arg1 context.Context, ar arg4 bool arg5 bool arg6 vpc.Importer - }{arg1, arg2Copy, arg3, arg4, arg5, arg6}) + arg7 int + }{arg1, arg2Copy, arg3, arg4, arg5, arg6, arg7}) stub := fake.NewUnmanagedNodeGroupTaskStub fakeReturns := fake.newUnmanagedNodeGroupTaskReturns - fake.recordInvocation("NewUnmanagedNodeGroupTask", []interface{}{arg1, arg2Copy, arg3, arg4, arg5, arg6}) + fake.recordInvocation("NewUnmanagedNodeGroupTask", []interface{}{arg1, arg2Copy, arg3, arg4, arg5, arg6, arg7}) fake.newUnmanagedNodeGroupTaskMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4, arg5, arg6) + return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } if specificReturn { return ret.result1 @@ -4410,17 +4416,17 @@ func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskCallCount() int { return len(fake.newUnmanagedNodeGroupTaskArgsForCall) } -func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskCalls(stub func(context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer) *tasks.TaskTree) { +func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskCalls(stub func(context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer, int) *tasks.TaskTree) { fake.newUnmanagedNodeGroupTaskMutex.Lock() defer fake.newUnmanagedNodeGroupTaskMutex.Unlock() fake.NewUnmanagedNodeGroupTaskStub = stub } -func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskArgsForCall(i int) (context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer) { +func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskArgsForCall(i int) (context.Context, []*v1alpha5.NodeGroup, bool, bool, bool, vpc.Importer, int) { fake.newUnmanagedNodeGroupTaskMutex.RLock() defer fake.newUnmanagedNodeGroupTaskMutex.RUnlock() argsForCall := fake.newUnmanagedNodeGroupTaskArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } func (fake *FakeStackManager) NewUnmanagedNodeGroupTaskReturns(result1 *tasks.TaskTree) { diff --git a/pkg/cfn/manager/interface.go b/pkg/cfn/manager/interface.go index a233f0ceb0..ef887db7bc 100644 --- a/pkg/cfn/manager/interface.go +++ b/pkg/cfn/manager/interface.go @@ -84,15 +84,15 @@ type StackManager interface { LookupCloudTrailEvents(ctx context.Context, i *Stack) ([]cttypes.Event, error) MakeChangeSetName(action string) string MakeClusterStackName() string - NewManagedNodeGroupTask(ctx context.Context, nodeGroups []*api.ManagedNodeGroup, forceAddCNIPolicy bool, importer vpc.Importer) *tasks.TaskTree + NewManagedNodeGroupTask(ctx context.Context, nodeGroups []*api.ManagedNodeGroup, forceAddCNIPolicy bool, importer vpc.Importer, nodeGroupParallelism int) *tasks.TaskTree NewTasksToDeleteClusterWithNodeGroups(ctx context.Context, clusterStack *Stack, nodeGroupStacks []NodeGroupStack, clusterOperable bool, newOIDCManager NewOIDCManager, newTasksToDeleteAddonIAM NewTasksToDeleteAddonIAM, newTasksToDeletePodIdentityRole NewTasksToDeletePodIdentityRole, cluster *ekstypes.Cluster, clientSetGetter kubernetes.ClientSetGetter, wait, force bool, cleanup func(chan error, string) error) (*tasks.TaskTree, error) NewTasksToCreateIAMServiceAccounts(serviceAccounts []*api.ClusterIAMServiceAccount, oidc *iamoidc.OpenIDConnectManager, clientSetGetter kubernetes.ClientSetGetter) *tasks.TaskTree NewTaskToDeleteUnownedNodeGroup(ctx context.Context, clusterName, nodegroup string, nodeGroupDeleter NodeGroupDeleter, waitCondition *DeleteWaitCondition) tasks.Task - NewTasksToCreateCluster(ctx context.Context, nodeGroups []*api.NodeGroup, managedNodeGroups []*api.ManagedNodeGroup, accessConfig *api.AccessConfig, accessEntryCreator accessentry.CreatorInterface, postClusterCreationTasks ...tasks.Task) (*tasks.TaskTree, error) + NewTasksToCreateCluster(ctx context.Context, nodeGroups []*api.NodeGroup, managedNodeGroups []*api.ManagedNodeGroup, accessConfig *api.AccessConfig, accessEntryCreator accessentry.CreatorInterface, nodeGroupParallelism int, postClusterCreationTasks ...tasks.Task) *tasks.TaskTree NewTasksToDeleteIAMServiceAccounts(ctx context.Context, serviceAccounts []string, clientSetGetter kubernetes.ClientSetGetter, wait bool) (*tasks.TaskTree, error) NewTasksToDeleteNodeGroups(stacks []NodeGroupStack, shouldDelete func(_ string) bool, wait bool, cleanup func(chan error, string) error) (*tasks.TaskTree, error) NewTasksToDeleteOIDCProviderWithIAMServiceAccounts(ctx context.Context, newOIDCManager NewOIDCManager, cluster *ekstypes.Cluster, clientSetGetter kubernetes.ClientSetGetter, force bool) (*tasks.TaskTree, error) - NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, importer vpc.Importer) *tasks.TaskTree + NewUnmanagedNodeGroupTask(ctx context.Context, nodeGroups []*api.NodeGroup, forceAddCNIPolicy, skipEgressRules, disableAccessEntryCreation bool, importer vpc.Importer, nodeGroupParallelism int) *tasks.TaskTree PropagateManagedNodeGroupTagsToASG(ngName string, ngTags map[string]string, asgNames []string, errCh chan error) error RefreshFargatePodExecutionRoleARN(ctx context.Context) error StackStatusIsNotTransitional(s *Stack) bool diff --git a/pkg/cfn/manager/nodegroup.go b/pkg/cfn/manager/nodegroup.go index bf07fafebb..c31418f819 100644 --- a/pkg/cfn/manager/nodegroup.go +++ b/pkg/cfn/manager/nodegroup.go @@ -47,6 +47,7 @@ type CreateNodeGroupOptions struct { DisableAccessEntryCreation bool VPCImporter vpc.Importer SharedTags []types.Tag + Parallelism int } // A NodeGroupStackManager describes and creates nodegroup stacks. @@ -92,7 +93,7 @@ type OceanManagedNodeGroupTask struct { // Create creates a TaskTree for creating nodegroups. func (t *UnmanagedNodeGroupTask) Create(ctx context.Context, options CreateNodeGroupOptions) *tasks.TaskTree { - taskTree := &tasks.TaskTree{Parallel: true} + taskTree := &tasks.TaskTree{Parallel: true, Limit: options.Parallelism} for _, ng := range t.NodeGroups { ng := ng @@ -285,6 +286,7 @@ func (t *OceanManagedNodeGroupTask) maybeCreateAccessEntry(ctx context.Context, return fmt.Errorf("creating access entry for ocean nodegroup %s: %w", ng.Name, err) } logger.Info("ocean nodegroup %s: created access entry for principal ARN %q", ng.Name, roleARN) + return nil } diff --git a/pkg/cfn/manager/tasks_test.go b/pkg/cfn/manager/tasks_test.go index ea81694ae8..8edfa1a87e 100644 --- a/pkg/cfn/manager/tasks_test.go +++ b/pkg/cfn/manager/tasks_test.go @@ -80,22 +80,22 @@ var _ = Describe("StackCollection Tasks", func() { // The supportsManagedNodes argument has no effect on the Describe call, so the values are alternated // in these tests { - tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("bar", "foo"), false, false, true, fakeVPCImporter) + tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("bar", "foo"), false, false, true, fakeVPCImporter, 0) Expect(tasks.Describe()).To(Equal(` -2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" +2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" } `)) } { - tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("bar"), false, false, true, fakeVPCImporter) + tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("bar"), false, false, true, fakeVPCImporter, 0) Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "bar" }`)) } { - tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("foo"), false, false, true, fakeVPCImporter) + tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), makeNodeGroups("foo"), false, false, true, fakeVPCImporter, 0) Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "foo" }`)) } { - tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), nil, false, false, true, fakeVPCImporter) + tasks := stackManager.NewUnmanagedNodeGroupTask(context.Background(), nil, false, false, true, fakeVPCImporter, 0) Expect(tasks.Describe()).To(Equal(`no tasks`)) } @@ -103,86 +103,86 @@ var _ = Describe("StackCollection Tasks", func() { AuthenticationMode: ekstypes.AuthenticationModeConfigMap, } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), nil, accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), nil, accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", - 2 parallel sub-tasks: { +2 sequential tasks: { create cluster control plane "test-cluster", + 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", - } + } } `)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar"), nil, accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar"), nil, accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" +2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" } `)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), nil, nil, accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), nil, nil, accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(`1 task: { create cluster control plane "test-cluster" }`)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", - 2 parallel sub-tasks: { - 2 parallel sub-tasks: { +2 sequential tasks: { create cluster control plane "test-cluster", + 2 parallel sub-tasks: { + 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", }, - 2 parallel sub-tasks: { + 2 parallel sub-tasks: { create managed nodegroup "m1", create managed nodegroup "m2", }, - } + } } `)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), makeManagedNodeGroupsWithPropagatedTags("m1", "m2"), accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar", "foo"), makeManagedNodeGroupsWithPropagatedTags("m1", "m2"), accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", - 2 parallel sub-tasks: { - 2 parallel sub-tasks: { +2 sequential tasks: { create cluster control plane "test-cluster", + 2 parallel sub-tasks: { + 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", }, - 2 parallel sub-tasks: { - 2 sequential sub-tasks: { + 2 parallel sub-tasks: { + 2 sequential sub-tasks: { create managed nodegroup "m1", propagate tags to ASG for managed nodegroup "m1", }, - 2 sequential sub-tasks: { + 2 sequential sub-tasks: { create managed nodegroup "m2", propagate tags to ASG for managed nodegroup "m2", }, }, - } + } } `)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("foo"), makeManagedNodeGroups("m1"), accessConfig, nil) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("foo"), makeManagedNodeGroups("m1"), accessConfig, nil, 0) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", - 2 parallel sub-tasks: { +2 sequential tasks: { create cluster control plane "test-cluster", + 2 parallel sub-tasks: { create nodegroup "foo", create managed nodegroup "m1", - } + } } `)) } { - tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar"), nil, accessConfig, nil, &task{id: 1}) + tasks := stackManager.NewTasksToCreateCluster(context.Background(), makeNodeGroups("bar"), nil, accessConfig, nil, 0, &task{id: 1}) Expect(tasks.Describe()).To(Equal(` -2 sequential tasks: { create cluster control plane "test-cluster", - 2 sequential sub-tasks: { +2 sequential tasks: { create cluster control plane "test-cluster", + 2 sequential sub-tasks: { task 1, create nodegroup "bar", - } + } } `)) } @@ -203,20 +203,20 @@ var _ = Describe("StackCollection Tasks", func() { stackManager = NewStackCollection(p, cfg) }) It("returns an error", func() { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{}, nil) + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{}, nil) ng := api.NewManagedNodeGroup() fakeVPCImporter := new(vpcfakes.FakeImporter) - tasks := stackManager.NewManagedNodeGroupTask(context.Background(), []*api.ManagedNodeGroup{ng}, false, fakeVPCImporter) + tasks := stackManager.NewManagedNodeGroupTask(context.Background(), []*api.ManagedNodeGroup{ng}, false, fakeVPCImporter, 0) errs := tasks.DoAllSync() Expect(errs).To(HaveLen(1)) Expect(errs[0]).To(MatchError(ContainSubstring("managed nodegroups cannot be created on IPv6 unowned clusters"))) }) When("finding the stack fails", func() { It("returns the stack error", func() { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(nil, errors.New("not found")) + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("not found")) ng := api.NewManagedNodeGroup() fakeVPCImporter := new(vpcfakes.FakeImporter) - tasks := stackManager.NewManagedNodeGroupTask(context.Background(), []*api.ManagedNodeGroup{ng}, false, fakeVPCImporter) + tasks := stackManager.NewManagedNodeGroupTask(context.Background(), []*api.ManagedNodeGroup{ng}, false, fakeVPCImporter, 0) errs := tasks.DoAllSync() Expect(errs).To(HaveLen(1)) Expect(errs[0]).To(MatchError(ContainSubstring("not found"))) @@ -253,7 +253,7 @@ var _ = Describe("StackCollection Tasks", func() { }, Entry("an OIDC provider is associated with the cluster", oidcEntry{ mockProvider: func(p *mockprovider.MockProvider) { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cfntypes.StackSummary{}, }, nil) }, @@ -268,7 +268,7 @@ var _ = Describe("StackCollection Tasks", func() { Entry("cluster has IAM service accounts", oidcEntry{ mockProvider: func(p *mockprovider.MockProvider) { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cfntypes.StackSummary{ { StackName: aws.String("eksctl-test-cluster-addon-iamserviceaccount-test"), @@ -298,7 +298,7 @@ var _ = Describe("StackCollection Tasks", func() { Entry("OIDC provider and service accounts do not exist for the cluster", oidcEntry{ mockProvider: func(p *mockprovider.MockProvider) { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cfntypes.StackSummary{}, }, nil) }, @@ -309,7 +309,7 @@ var _ = Describe("StackCollection Tasks", func() { Entry("OIDC provider definitely does not exist for the cluster", oidcEntry{ mockProvider: func(p *mockprovider.MockProvider) { - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cfntypes.StackSummary{}, }, nil) }, diff --git a/pkg/ctl/cmdutils/configfile.go b/pkg/ctl/cmdutils/configfile.go index 76cbbac34b..a87645db59 100644 --- a/pkg/ctl/cmdutils/configfile.go +++ b/pkg/ctl/cmdutils/configfile.go @@ -3,6 +3,7 @@ package cmdutils import ( "encoding/csv" "fmt" + "io" "reflect" "strconv" "strings" @@ -36,6 +37,7 @@ type ClusterConfigLoader interface { type commonClusterConfigLoader struct { *Cmd + configReader io.Reader flagsIncompatibleWithConfigFile sets.Set[string] flagsIncompatibleWithoutConfigFile sets.Set[string] @@ -129,7 +131,7 @@ func (l *commonClusterConfigLoader) Load() error { // The reference to ClusterConfig should only be reassigned if ClusterConfigFile is specified // because other parts of the code store the pointer locally and access it directly instead of via // the Cmd reference - if l.ClusterConfig, err = eks.LoadConfigFromFile(l.ClusterConfigFile); err != nil { + if l.ClusterConfig, err = eks.LoadConfigWithReader(l.ClusterConfigFile, l.configReader); err != nil { return err } meta := l.ClusterConfig.Metadata @@ -203,6 +205,7 @@ func NewMetadataLoader(cmd *Cmd) ClusterConfigLoader { // NewCreateClusterLoader will load config or use flags for 'eksctl create cluster' func NewCreateClusterLoader(cmd *Cmd, ngFilter *filter.NodeGroupFilter, ng *api.NodeGroup, params *CreateClusterCmdParams) ClusterConfigLoader { l := newCommonClusterConfigLoader(cmd) + l.configReader = params.ConfigReader ngFilter.SetExcludeAll(params.WithoutNodeGroup) @@ -313,6 +316,10 @@ func NewCreateClusterLoader(cmd *Cmd, ngFilter *filter.NodeGroupFilter, ng *api. } } + if err := validateBareCluster(clusterConfig); err != nil { + return err + } + shallCreatePodIdentityAssociations := func(cfg *api.ClusterConfig) bool { if cfg.IAM != nil && len(cfg.IAM.PodIdentityAssociations) > 0 { return true @@ -452,6 +459,22 @@ func validateDryRunOptions(cmd *cobra.Command, incompatibleFlags []string) error return nil } +// validateBareCluster validates a cluster for unsupported fields if VPC CNI is disabled. +func validateBareCluster(clusterConfig *api.ClusterConfig) error { + if !clusterConfig.AddonsConfig.DisableDefaultAddons || slices.ContainsFunc(clusterConfig.Addons, func(addon *api.Addon) bool { + return addon.Name == api.VPCCNIAddon + }) { + return nil + } + if clusterConfig.HasNodes() || clusterConfig.IsFargateEnabled() || clusterConfig.Karpenter != nil || clusterConfig.HasGitOpsFluxConfigured() || + (clusterConfig.IAM != nil && ((len(clusterConfig.IAM.ServiceAccounts) > 0) || len(clusterConfig.IAM.PodIdentityAssociations) > 0)) { + return errors.New("fields nodeGroups, managedNodeGroups, fargateProfiles, karpenter, gitops, iam.serviceAccounts, " + + "and iam.podIdentityAssociations are not supported during cluster creation in a cluster without VPC CNI; please remove these fields " + + "and add them back after cluster creation is successful") + } + return nil +} + const updateAuthConfigMapFlagName = "update-auth-configmap" // NewCreateNodeGroupLoader will load config or use flags for 'eksctl create nodegroup' diff --git a/pkg/ctl/cmdutils/configfile_test.go b/pkg/ctl/cmdutils/configfile_test.go index 70b5b58d68..e476fed432 100644 --- a/pkg/ctl/cmdutils/configfile_test.go +++ b/pkg/ctl/cmdutils/configfile_test.go @@ -3,11 +3,13 @@ package cmdutils import ( "path/filepath" + "github.com/aws/aws-sdk-go-v2/aws" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/cobra" "github.com/spf13/pflag" + clusterutils "github.com/weaveworks/eksctl/integration/utilities/cluster" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/ctl/cmdutils/filter" ) @@ -471,6 +473,123 @@ var _ = Describe("cmdutils configfile", func() { testClusterEndpointAccessDefaults("test_data/cluster-with-vpc-private-access.yaml", true, true) }) }) + + type bareClusterEntry struct { + updateClusterConfig func(*api.ClusterConfig) + expectErr bool + } + + DescribeTable("Bare Cluster validation", func(e bareClusterEntry) { + cmd := &Cmd{ + CobraCommand: newCmd(), + ClusterConfigFile: "-", + ClusterConfig: api.NewClusterConfig(), + ProviderConfig: api.ProviderConfig{}, + } + clusterConfig := api.NewClusterConfig() + clusterConfig.Metadata.Name = "cluster" + clusterConfig.Metadata.Region = api.DefaultRegion + clusterConfig.AddonsConfig.DisableDefaultAddons = true + clusterConfig.Addons = []*api.Addon{ + { + Name: api.CoreDNSAddon, + }, + } + e.updateClusterConfig(clusterConfig) + err := NewCreateClusterLoader(cmd, filter.NewNodeGroupFilter(), nil, &CreateClusterCmdParams{ + ConfigReader: clusterutils.Reader(clusterConfig), + }).Load() + if e.expectErr { + Expect(err).To(MatchError("fields nodeGroups, managedNodeGroups, fargateProfiles, karpenter, gitops, iam.serviceAccounts, " + + "and iam.podIdentityAssociations are not supported during cluster creation in a cluster without VPC CNI; please remove these fields " + + "and add them back after cluster creation is successful")) + } else { + Expect(err).NotTo(HaveOccurred()) + } + }, + Entry("nodeGroups", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + ng := api.NewNodeGroup() + ng.Name = "ng" + ng.DesiredCapacity = aws.Int(1) + c.NodeGroups = []*api.NodeGroup{ng} + }, + expectErr: true, + }), + Entry("managedNodeGroups", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + ng := api.NewManagedNodeGroup() + ng.Name = "mng" + ng.DesiredCapacity = aws.Int(1) + c.ManagedNodeGroups = []*api.ManagedNodeGroup{ng} + }, + expectErr: true, + }), + Entry("fargateProfiles", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + c.FargateProfiles = []*api.FargateProfile{ + { + Name: "test", + Selectors: []api.FargateProfileSelector{ + { + Namespace: "default", + }, + }, + }, + } + }, + expectErr: true, + }), + Entry("gitops", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + c.GitOps = &api.GitOps{ + Flux: &api.Flux{ + GitProvider: "github", + Flags: api.FluxFlags{ + "owner": "aws", + }, + }, + } + }, + expectErr: true, + }), + Entry("karpenter", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + c.Karpenter = &api.Karpenter{} + }, + expectErr: true, + }), + Entry("iam.serviceAccounts", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + c.IAM.WithOIDC = api.Enabled() + c.IAM.ServiceAccounts = []*api.ClusterIAMServiceAccount{ + { + ClusterIAMMeta: api.ClusterIAMMeta{ + Name: "test", + Namespace: "test", + }, + AttachPolicyARNs: []string{"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"}, + }, + } + }, + expectErr: true, + }), + Entry("iam.podIdentityAssociations", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) { + c.Addons = append(c.Addons, &api.Addon{Name: api.PodIdentityAgentAddon}) + c.IAM.PodIdentityAssociations = []api.PodIdentityAssociation{ + { + Namespace: "test", + PermissionPolicyARNs: []string{"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"}, + }, + } + }, + expectErr: true, + }), + Entry("no unsupported field set", bareClusterEntry{ + updateClusterConfig: func(c *api.ClusterConfig) {}, + }), + ) }) Describe("SetLabelLoader", func() { diff --git a/pkg/ctl/cmdutils/params.go b/pkg/ctl/cmdutils/create_cluster.go similarity index 97% rename from pkg/ctl/cmdutils/params.go rename to pkg/ctl/cmdutils/create_cluster.go index 92abb74b49..683ba740ae 100644 --- a/pkg/ctl/cmdutils/params.go +++ b/pkg/ctl/cmdutils/create_cluster.go @@ -1,6 +1,7 @@ package cmdutils import ( + "io" "time" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" @@ -24,6 +25,7 @@ type CreateClusterCmdParams struct { CreateNGOptions CreateManagedNGOptions CreateSpotOceanNodeGroupOptions + ConfigReader io.Reader } // NodeGroupOptions holds options for creating nodegroups. @@ -48,6 +50,7 @@ type CreateNGOptions struct { InstallNeuronDevicePlugin bool InstallNvidiaDevicePlugin bool DryRun bool + NodeGroupParallelism int } // CreateSpotOceanNodeGroupOptions holds options for creating a Spot Ocean nodegroup. diff --git a/pkg/ctl/cmdutils/nodegroup_flags.go b/pkg/ctl/cmdutils/nodegroup_flags.go index 574d3edb69..8f9b50d9ca 100644 --- a/pkg/ctl/cmdutils/nodegroup_flags.go +++ b/pkg/ctl/cmdutils/nodegroup_flags.go @@ -64,6 +64,7 @@ func AddCommonCreateNodeGroupAddonsFlags(fs *pflag.FlagSet, ng *api.NodeGroup, o addCommonCreateNodeGroupIAMAddonsFlags(fs, ng) fs.BoolVarP(&options.InstallNeuronDevicePlugin, "install-neuron-plugin", "", true, "install Neuron plugin for Inferentia and Trainium nodes") fs.BoolVarP(&options.InstallNvidiaDevicePlugin, "install-nvidia-plugin", "", true, "install Nvidia plugin for GPU nodes") + fs.IntVarP(&options.NodeGroupParallelism, "nodegroup-parallelism", "", 8, "Number of self-managed or managed nodegroups to create in parallel") } // AddInstanceSelectorOptions adds flags for EC2 instance selector diff --git a/pkg/ctl/cmdutils/zonal_shift_config.go b/pkg/ctl/cmdutils/zonal_shift_config.go new file mode 100644 index 0000000000..999b849199 --- /dev/null +++ b/pkg/ctl/cmdutils/zonal_shift_config.go @@ -0,0 +1,33 @@ +package cmdutils + +import ( + "errors" + "fmt" +) + +// NewZonalShiftConfigLoader creates a new loader for zonal shift config. +func NewZonalShiftConfigLoader(cmd *Cmd) ClusterConfigLoader { + l := newCommonClusterConfigLoader(cmd) + l.flagsIncompatibleWithConfigFile.Insert( + "enable-zonal-shift", + "cluster", + ) + + l.validateWithConfigFile = func() error { + if cmd.NameArg != "" { + return fmt.Errorf("config file and enable-zonal-shift %s", IncompatibleFlags) + } + if l.ClusterConfig.ZonalShiftConfig == nil || l.ClusterConfig.ZonalShiftConfig.Enabled == nil { + return errors.New("field zonalShiftConfig.enabled is required") + } + return nil + } + + l.validateWithoutConfigFile = func() error { + if !cmd.CobraCommand.Flag("enable-zonal-shift").Changed { + return errors.New("--enable-zonal-shift is required when a config file is not specified") + } + return nil + } + return l +} diff --git a/pkg/ctl/create/cluster.go b/pkg/ctl/create/cluster.go index 21d9ebc4a7..351f0673cf 100644 --- a/pkg/ctl/create/cluster.go +++ b/pkg/ctl/create/cluster.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os/exec" + "strings" "sync" "github.com/aws/aws-sdk-go-v2/aws" @@ -37,7 +38,6 @@ import ( "github.com/weaveworks/eksctl/pkg/utils/kubeconfig" "github.com/weaveworks/eksctl/pkg/utils/names" "github.com/weaveworks/eksctl/pkg/utils/nodes" - "github.com/weaveworks/eksctl/pkg/utils/tasks" "github.com/weaveworks/eksctl/pkg/vpc" ) @@ -357,22 +357,18 @@ func doCreateCluster(cmd *cmdutils.Cmd, ngFilter *filter.NodeGroupFilter, params logger.Info("if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=%s --cluster=%s'", meta.Region, meta.Name) eks.LogEnabledFeatures(cfg) - postClusterCreationTasks := ctl.CreateExtraClusterConfigTasks(ctx, cfg) - var preNodegroupAddons, postNodegroupAddons *tasks.TaskTree - if len(cfg.Addons) > 0 { - iamRoleCreator := &podidentityassociation.IAMRoleCreator{ - ClusterName: cfg.Metadata.Name, - StackCreator: stackManager, - } - preNodegroupAddons, postNodegroupAddons = addon.CreateAddonTasks(ctx, cfg, ctl, iamRoleCreator, true, cmd.ProviderConfig.WaitTimeout) - postClusterCreationTasks.Append(preNodegroupAddons) + iamRoleCreator := &podidentityassociation.IAMRoleCreator{ + ClusterName: cfg.Metadata.Name, + StackCreator: stackManager, } - - taskTree, err := stackManager.NewTasksToCreateCluster(ctx, cfg.NodeGroups, cfg.ManagedNodeGroups, cfg.AccessConfig, makeAccessEntryCreator(cfg.Metadata.Name, stackManager), postClusterCreationTasks) - if err != nil { - return fmt.Errorf("ocean: failed to create cluster nodegroup: %v", err) + preNodegroupAddons, postNodegroupAddons, updateVPCCNITask, autoDefaultAddons := addon.CreateAddonTasks(ctx, cfg, ctl, iamRoleCreator, true, cmd.ProviderConfig.WaitTimeout) + if len(autoDefaultAddons) > 0 { + logger.Info("default addons %s were not specified, will install them as EKS addons", strings.Join(autoDefaultAddons, ", ")) } + postClusterCreationTasks := ctl.CreateExtraClusterConfigTasks(ctx, cfg, preNodegroupAddons, updateVPCCNITask) + + taskTree := stackManager.NewTasksToCreateCluster(ctx, cfg.NodeGroups, cfg.ManagedNodeGroups, cfg.AccessConfig, makeAccessEntryCreator(cfg.Metadata.Name, stackManager), params.NodeGroupParallelism, postClusterCreationTasks) // Spot Ocean. { @@ -462,7 +458,7 @@ func doCreateCluster(cmd *cmdutils.Cmd, ngFilter *filter.NodeGroupFilter, params // authorize self-managed nodes to join the cluster via aws-auth configmap // only if EKS access entries are disabled if cfg.AccessConfig.AuthenticationMode == ekstypes.AuthenticationModeConfigMap { - if err := eks.UpdateAuthConfigMap(ngCtx, cfg.NodeGroups, clientSet); err != nil { + if err := eks.UpdateAuthConfigMap(cfg.NodeGroups, clientSet); err != nil { return err } } diff --git a/pkg/ctl/create/cluster_test.go b/pkg/ctl/create/cluster_test.go index 4c2d388771..f876004ca4 100644 --- a/pkg/ctl/create/cluster_test.go +++ b/pkg/ctl/create/cluster_test.go @@ -273,6 +273,7 @@ var _ = Describe("create cluster", func() { clusterConfig := api.NewClusterConfig() clusterConfig.Metadata.Name = clusterName + clusterConfig.AddonsConfig.DisableDefaultAddons = true clusterConfig.VPC.ClusterEndpoints = api.ClusterEndpointAccessDefaults() clusterConfig.AccessConfig.AuthenticationMode = ekstypes.AuthenticationModeApiAndConfigMap @@ -877,7 +878,7 @@ var ( updateMocksForNodegroups = func(status cftypes.StackStatus, outputs []cftypes.Output) func(mp *mockprovider.MockProvider) { return func(mp *mockprovider.MockProvider) { - mp.MockEC2().On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + mp.MockEC2().On("DescribeInstanceTypeOfferings", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { InstanceType: "g3.xlarge", @@ -951,7 +952,7 @@ func defaultProviderMocks(p *mockprovider.MockProvider, output []cftypes.Output, ZoneId: aws.String("id"), }}, }, nil) - p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ + p.MockCloudFormation().On("ListStacks", mock.Anything, mock.Anything, mock.Anything).Return(&cloudformation.ListStacksOutput{ StackSummaries: []cftypes.StackSummary{ { StackName: aws.String(clusterStackName), @@ -1039,7 +1040,7 @@ func defaultProviderMocks(p *mockprovider.MockProvider, output []cftypes.Output, }, }, }, nil) - p.MockEC2().On("DescribeSubnets", mock.Anything, mock.Anything).Return(&ec2.DescribeSubnetsOutput{ + p.MockEC2().On("DescribeSubnets", mock.Anything, mock.Anything, mock.Anything).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{}, }, nil) p.MockEC2().On("DescribeVpcs", mock.Anything, mock.Anything).Return(&ec2.DescribeVpcsOutput{ @@ -1075,7 +1076,7 @@ func mockOutposts(provider *mockprovider.MockProvider, outpostID string) { }, nil) provider.MockOutposts().On("GetOutpostInstanceTypes", mock.Anything, &outposts.GetOutpostInstanceTypesInput{ OutpostId: aws.String(outpostID), - }).Return(&outposts.GetOutpostInstanceTypesOutput{ + }, mock.Anything).Return(&outposts.GetOutpostInstanceTypesOutput{ InstanceTypes: []outpoststypes.InstanceTypeItem{ { InstanceType: aws.String("m5.xlarge"), @@ -1084,7 +1085,7 @@ func mockOutposts(provider *mockprovider.MockProvider, outpostID string) { }, nil) provider.MockEC2().On("DescribeInstanceTypes", mock.Anything, &ec2.DescribeInstanceTypesInput{ InstanceTypes: []ec2types.InstanceType{"m5.xlarge"}, - }).Return(&ec2.DescribeInstanceTypesOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: []ec2types.InstanceTypeInfo{ { InstanceType: "m5.xlarge", diff --git a/pkg/ctl/create/nodegroup.go b/pkg/ctl/create/nodegroup.go index e316c9b405..b771b6e6db 100644 --- a/pkg/ctl/create/nodegroup.go +++ b/pkg/ctl/create/nodegroup.go @@ -82,6 +82,7 @@ func createNodeGroupCmd(cmd *cmdutils.Cmd) { }, SkipOutdatedAddonsCheck: options.SkipOutdatedAddonsCheck, ConfigFileProvided: cmd.ClusterConfigFile != "", + Parallelism: options.NodeGroupParallelism, }, ngFilter) }) } diff --git a/pkg/ctl/utils/update_addon.go b/pkg/ctl/utils/update_addon.go new file mode 100644 index 0000000000..b10e50ae2c --- /dev/null +++ b/pkg/ctl/utils/update_addon.go @@ -0,0 +1,53 @@ +package utils + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/eks" + + defaultaddons "github.com/weaveworks/eksctl/pkg/addons/default" + "github.com/weaveworks/eksctl/pkg/ctl/cmdutils" + "github.com/weaveworks/eksctl/pkg/kubernetes" + "github.com/weaveworks/eksctl/pkg/utils/apierrors" +) + +type handleAddonUpdate func(*kubernetes.RawClient, defaultaddons.AddonVersionDescriber) (updateRequired bool, err error) + +func updateAddon(ctx context.Context, cmd *cmdutils.Cmd, addonName string, handleUpdate handleAddonUpdate) error { + if err := cmdutils.NewMetadataLoader(cmd).Load(); err != nil { + return err + } + ctl, err := cmd.NewProviderForExistingCluster(ctx) + if err != nil { + return err + } + if ok, err := ctl.CanUpdate(cmd.ClusterConfig); !ok { + return err + } + + eksAPI := ctl.AWSProvider.EKS() + switch _, err := eksAPI.DescribeAddon(ctx, &eks.DescribeAddonInput{ + AddonName: aws.String(addonName), + ClusterName: aws.String(cmd.ClusterConfig.Metadata.Name), + }); { + case err == nil: + return fmt.Errorf("addon %s is installed as a managed EKS addon; to update it, use `eksctl update addon` instead", addonName) + case apierrors.IsNotFoundError(err): + + default: + return fmt.Errorf("error describing addon %s: %w", addonName, err) + } + + rawClient, err := ctl.NewRawClient(cmd.ClusterConfig) + if err != nil { + return err + } + updateRequired, err := handleUpdate(rawClient, eksAPI) + if err != nil { + return err + } + cmdutils.LogPlanModeWarning(cmd.Plan && updateRequired) + return nil +} diff --git a/pkg/ctl/utils/update_aws_node.go b/pkg/ctl/utils/update_aws_node.go index 4dadc1aec6..6e53d0aede 100644 --- a/pkg/ctl/utils/update_aws_node.go +++ b/pkg/ctl/utils/update_aws_node.go @@ -9,6 +9,7 @@ import ( defaultaddons "github.com/weaveworks/eksctl/pkg/addons/default" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/ctl/cmdutils" + "github.com/weaveworks/eksctl/pkg/kubernetes" ) func updateAWSNodeCmd(cmd *cmdutils.Cmd) { @@ -34,37 +35,11 @@ func updateAWSNodeCmd(cmd *cmdutils.Cmd) { } func doUpdateAWSNode(cmd *cmdutils.Cmd) error { - if err := cmdutils.NewMetadataLoader(cmd).Load(); err != nil { - return err - } - - cfg := cmd.ClusterConfig - meta := cmd.ClusterConfig.Metadata - ctx := context.TODO() - ctl, err := cmd.NewProviderForExistingCluster(ctx) - if err != nil { - return err - } - - if ok, err := ctl.CanUpdate(cfg); !ok { - return err - } - - rawClient, err := ctl.NewRawClient(cfg) - if err != nil { - return err - } - - updateRequired, err := defaultaddons.UpdateAWSNode(ctx, defaultaddons.AddonInput{ - RawClient: rawClient, - Region: meta.Region, - }, cmd.Plan) - if err != nil { - return err - } - - cmdutils.LogPlanModeWarning(cmd.Plan && updateRequired) - - return nil + return updateAddon(ctx, cmd, api.VPCCNIAddon, func(rawClient *kubernetes.RawClient, _ defaultaddons.AddonVersionDescriber) (bool, error) { + return defaultaddons.UpdateAWSNode(ctx, defaultaddons.AddonInput{ + RawClient: rawClient, + Region: cmd.ClusterConfig.Metadata.Region, + }, cmd.Plan) + }) } diff --git a/pkg/ctl/utils/update_coredns.go b/pkg/ctl/utils/update_coredns.go index 3728f2a32c..5e37fb2f02 100644 --- a/pkg/ctl/utils/update_coredns.go +++ b/pkg/ctl/utils/update_coredns.go @@ -9,6 +9,7 @@ import ( defaultaddons "github.com/weaveworks/eksctl/pkg/addons/default" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/ctl/cmdutils" + "github.com/weaveworks/eksctl/pkg/kubernetes" ) func updateCoreDNSCmd(cmd *cmdutils.Cmd) { @@ -34,44 +35,16 @@ func updateCoreDNSCmd(cmd *cmdutils.Cmd) { } func doUpdateCoreDNS(cmd *cmdutils.Cmd) error { - if err := cmdutils.NewMetadataLoader(cmd).Load(); err != nil { - return err - } - - cfg := cmd.ClusterConfig - meta := cmd.ClusterConfig.Metadata - ctx := context.TODO() - ctl, err := cmd.NewProviderForExistingCluster(ctx) - if err != nil { - return err - } - - if ok, err := ctl.CanUpdate(cfg); !ok { - return err - } - - rawClient, err := ctl.NewRawClient(cfg) - if err != nil { - return err - } - - kubernetesVersion, err := rawClient.ServerVersion() - if err != nil { - return err - } - - updateRequired, err := defaultaddons.UpdateCoreDNS(ctx, defaultaddons.AddonInput{ - RawClient: rawClient, - ControlPlaneVersion: kubernetesVersion, - Region: meta.Region, - }, cmd.Plan) - - if err != nil { - return err - } - - cmdutils.LogPlanModeWarning(cmd.Plan && updateRequired) - - return nil + return updateAddon(ctx, cmd, api.CoreDNSAddon, func(rawClient *kubernetes.RawClient, _ defaultaddons.AddonVersionDescriber) (bool, error) { + kubernetesVersion, err := rawClient.ServerVersion() + if err != nil { + return false, err + } + return defaultaddons.UpdateCoreDNS(ctx, defaultaddons.AddonInput{ + RawClient: rawClient, + ControlPlaneVersion: kubernetesVersion, + Region: cmd.ClusterConfig.Metadata.Region, + }, cmd.Plan) + }) } diff --git a/pkg/ctl/utils/update_kube_proxy.go b/pkg/ctl/utils/update_kube_proxy.go index 9e9a390315..a6681406af 100644 --- a/pkg/ctl/utils/update_kube_proxy.go +++ b/pkg/ctl/utils/update_kube_proxy.go @@ -9,6 +9,7 @@ import ( defaultaddons "github.com/weaveworks/eksctl/pkg/addons/default" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/ctl/cmdutils" + "github.com/weaveworks/eksctl/pkg/kubernetes" ) func updateKubeProxyCmd(cmd *cmdutils.Cmd) { @@ -34,44 +35,17 @@ func updateKubeProxyCmd(cmd *cmdutils.Cmd) { } func doUpdateKubeProxy(cmd *cmdutils.Cmd) error { - if err := cmdutils.NewMetadataLoader(cmd).Load(); err != nil { - return err - } - - cfg := cmd.ClusterConfig - meta := cmd.ClusterConfig.Metadata - ctx := context.TODO() - ctl, err := cmd.NewProviderForExistingCluster(ctx) - if err != nil { - return err - } - - if ok, err := ctl.CanUpdate(cfg); !ok { - return err - } - - rawClient, err := ctl.NewRawClient(cfg) - if err != nil { - return err - } - - kubernetesVersion, err := rawClient.ServerVersion() - if err != nil { - return err - } - - updateRequired, err := defaultaddons.UpdateKubeProxy(ctx, defaultaddons.AddonInput{ - RawClient: rawClient, - ControlPlaneVersion: kubernetesVersion, - Region: meta.Region, - EKSAPI: ctl.AWSProvider.EKS(), - }, cmd.Plan) - if err != nil { - return err - } - - cmdutils.LogPlanModeWarning(cmd.Plan && updateRequired) - - return nil + return updateAddon(ctx, cmd, api.KubeProxyAddon, func(rawClient *kubernetes.RawClient, addonDescriber defaultaddons.AddonVersionDescriber) (bool, error) { + kubernetesVersion, err := rawClient.ServerVersion() + if err != nil { + return false, err + } + return defaultaddons.UpdateKubeProxy(ctx, defaultaddons.AddonInput{ + RawClient: rawClient, + ControlPlaneVersion: kubernetesVersion, + Region: cmd.ClusterConfig.Metadata.Region, + AddonVersionDescriber: addonDescriber, + }, cmd.Plan) + }) } diff --git a/pkg/ctl/utils/update_zonal_shift_config.go b/pkg/ctl/utils/update_zonal_shift_config.go new file mode 100644 index 0000000000..701f0c99f9 --- /dev/null +++ b/pkg/ctl/utils/update_zonal_shift_config.go @@ -0,0 +1,84 @@ +package utils + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/eks" + ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" + + "github.com/kris-nova/logger" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/ctl/cmdutils" +) + +func updateZonalShiftConfig(cmd *cmdutils.Cmd, handler func(*cmdutils.Cmd) error) { + cfg := api.NewClusterConfig() + cmd.ClusterConfig = cfg + + cmd.SetDescription("update-zonal-shift-config", "update zonal shift config", "update zonal shift config on a cluster") + + var enableZonalShift bool + cmd.CobraCommand.RunE = func(_ *cobra.Command, args []string) error { + cmd.NameArg = cmdutils.GetNameArg(args) + if err := cmdutils.NewZonalShiftConfigLoader(cmd).Load(); err != nil { + return err + } + if cmd.ClusterConfigFile == "" { + cfg.ZonalShiftConfig = &api.ZonalShiftConfig{ + Enabled: &enableZonalShift, + } + } + return handler(cmd) + } + + cmdutils.AddCommonFlagsForAWS(cmd, &cmd.ProviderConfig, false) + + cmd.FlagSetGroup.InFlagSet("General", func(fs *pflag.FlagSet) { + cmdutils.AddClusterFlag(fs, cfg.Metadata) + cmdutils.AddRegionFlag(fs, &cmd.ProviderConfig) + cmdutils.AddConfigFileFlag(fs, &cmd.ClusterConfigFile) + fs.BoolVar(&enableZonalShift, "enable-zonal-shift", true, "Enable zonal shift on a cluster") + }) + +} + +func updateZonalShiftConfigCmd(cmd *cmdutils.Cmd) { + updateZonalShiftConfig(cmd, doUpdateZonalShiftConfig) +} + +func doUpdateZonalShiftConfig(cmd *cmdutils.Cmd) error { + cfg := cmd.ClusterConfig + ctx := context.Background() + if cfg.Metadata.Name == "" { + return cmdutils.ErrMustBeSet(cmdutils.ClusterNameFlag(cmd)) + } + ctl, err := cmd.NewProviderForExistingCluster(ctx) + if err != nil { + return err + } + makeZonalShiftStatus := func(enabled *bool) string { + if api.IsEnabled(enabled) { + return "enabled" + } + return "disabled" + } + if zsc := ctl.Status.ClusterInfo.Cluster.ZonalShiftConfig; zsc != nil && *zsc.Enabled == api.IsEnabled(cfg.ZonalShiftConfig.Enabled) { + logger.Info("zonal shift is already %s", makeZonalShiftStatus(zsc.Enabled)) + return nil + } + if err := ctl.UpdateClusterConfig(ctx, &eks.UpdateClusterConfigInput{ + Name: aws.String(cfg.Metadata.Name), + ZonalShiftConfig: &ekstypes.ZonalShiftConfigRequest{ + Enabled: cfg.ZonalShiftConfig.Enabled, + }, + }); err != nil { + return fmt.Errorf("updating zonal shift config: %w", err) + } + logger.Info("zonal shift %s successfully", makeZonalShiftStatus(cfg.ZonalShiftConfig.Enabled)) + return nil +} diff --git a/pkg/ctl/utils/utils.go b/pkg/ctl/utils/utils.go index 85c6d7a022..29f396dfa3 100644 --- a/pkg/ctl/utils/utils.go +++ b/pkg/ctl/utils/utils.go @@ -33,6 +33,7 @@ func Command(flagGrouping *cmdutils.FlagGrouping) *cobra.Command { cmdutils.AddResourceCmd(flagGrouping, verbCmd, describeAddonConfigurationCmd) cmdutils.AddResourceCmd(flagGrouping, verbCmd, migrateToPodIdentityCmd) cmdutils.AddResourceCmd(flagGrouping, verbCmd, migrateAccessEntryCmd) + cmdutils.AddResourceCmd(flagGrouping, verbCmd, updateZonalShiftConfigCmd) return verbCmd } diff --git a/pkg/eks/api.go b/pkg/eks/api.go index 9c8e8582e5..e1ba830214 100644 --- a/pkg/eks/api.go +++ b/pkg/eks/api.go @@ -232,7 +232,12 @@ func ParseConfig(data []byte) (*api.ClusterConfig, error) { // LoadConfigFromFile loads ClusterConfig from configFile func LoadConfigFromFile(configFile string) (*api.ClusterConfig, error) { - data, err := readConfig(configFile) + return LoadConfigWithReader(configFile, nil) +} + +// LoadConfigWithReader loads ClusterConfig from configFile or configReader. +func LoadConfigWithReader(configFile string, configReader io.Reader) (*api.ClusterConfig, error) { + data, err := readConfig(configFile, configReader) if err != nil { return nil, errors.Wrapf(err, "reading config file %q", configFile) } @@ -241,12 +246,14 @@ func LoadConfigFromFile(configFile string) (*api.ClusterConfig, error) { return nil, errors.Wrapf(err, "loading config file %q", configFile) } return clusterConfig, nil - } -func readConfig(configFile string) ([]byte, error) { +func readConfig(configFile string, reader io.Reader) ([]byte, error) { if configFile == "-" { - return io.ReadAll(os.Stdin) + if reader == nil { + reader = os.Stdin + } + return io.ReadAll(reader) } return os.ReadFile(configFile) } diff --git a/pkg/eks/api_test.go b/pkg/eks/api_test.go index ffae2269bb..8eb683c08d 100644 --- a/pkg/eks/api_test.go +++ b/pkg/eks/api_test.go @@ -260,8 +260,8 @@ var _ = Describe("eksctl API", func() { }) - testEnsureAMI := func(matcher gomegatypes.GomegaMatcher) { - err := ResolveAMI(context.Background(), provider, "1.14", ng) + testEnsureAMI := func(matcher gomegatypes.GomegaMatcher, version string) { + err := ResolveAMI(context.Background(), provider, version, ng) ExpectWithOffset(1, err).NotTo(HaveOccurred()) ExpectWithOffset(1, ng.AMI).To(matcher) } @@ -275,7 +275,7 @@ var _ = Describe("eksctl API", func() { }, }, nil) - testEnsureAMI(Equal("ami-ssm")) + testEnsureAMI(Equal("ami-ssm"), "1.14") }) It("should fall back to auto resolution for Ubuntu1804", func() { @@ -283,7 +283,44 @@ var _ = Describe("eksctl API", func() { mockDescribeImages(provider, "ami-ubuntu", func(input *ec2.DescribeImagesInput) bool { return input.Owners[0] == "099720109477" }) - testEnsureAMI(Equal("ami-ubuntu")) + testEnsureAMI(Equal("ami-ubuntu"), "1.14") + }) + + It("should fall back to auto resolution for Ubuntu2004 on 1.14", func() { + ng.AMIFamily = api.NodeImageFamilyUbuntu2004 + mockDescribeImages(provider, "ami-ubuntu", func(input *ec2.DescribeImagesInput) bool { + return input.Owners[0] == "099720109477" + }) + testEnsureAMI(Equal("ami-ubuntu"), "1.14") + }) + + It("should resolve AMI using SSM Parameter Store for Ubuntu2004 on 1.29", func() { + provider.MockSSM().On("GetParameter", mock.Anything, &ssm.GetParameterInput{ + Name: aws.String("/aws/service/canonical/ubuntu/eks/20.04/1.29/stable/current/amd64/hvm/ebs-gp2/ami-id"), + }).Return(&ssm.GetParameterOutput{ + Parameter: &ssmtypes.Parameter{ + Value: aws.String("ami-ubuntu"), + }, + }, nil) + ng.AMIFamily = api.NodeImageFamilyUbuntu2004 + + testEnsureAMI(Equal("ami-ubuntu"), "1.29") + }) + + It("should fall back to auto resolution for Ubuntu2204", func() { + ng.AMIFamily = api.NodeImageFamilyUbuntu2204 + mockDescribeImages(provider, "ami-ubuntu", func(input *ec2.DescribeImagesInput) bool { + return input.Owners[0] == "099720109477" + }) + testEnsureAMI(Equal("ami-ubuntu"), "1.14") + }) + + It("should fall back to auto resolution for UbuntuPro2204", func() { + ng.AMIFamily = api.NodeImageFamilyUbuntuPro2204 + mockDescribeImages(provider, "ami-ubuntu", func(input *ec2.DescribeImagesInput) bool { + return input.Owners[0] == "099720109477" + }) + testEnsureAMI(Equal("ami-ubuntu"), "1.14") }) It("should fall back to auto resolution for Ubuntu2004", func() { @@ -317,7 +354,7 @@ var _ = Describe("eksctl API", func() { return len(input.ImageIds) == 0 }) - testEnsureAMI(Equal("ami-auto")) + testEnsureAMI(Equal("ami-auto"), "1.14") }) }) @@ -470,7 +507,7 @@ var _ = Describe("CheckInstanceAvailability", func() { }, LocationType: ec2types.LocationTypeAvailabilityZone, MaxResults: aws.Int32(100), - }).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { InstanceType: "t2.nano", @@ -610,7 +647,7 @@ var _ = Describe("CheckInstanceAvailability", func() { }, LocationType: ec2types.LocationTypeAvailabilityZone, MaxResults: aws.Int32(100), - }).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { InstanceType: "t2.nano", @@ -665,7 +702,7 @@ var _ = Describe("CheckInstanceAvailability", func() { }, LocationType: ec2types.LocationTypeAvailabilityZone, MaxResults: aws.Int32(100), - }).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: []ec2types.InstanceTypeOffering{ { InstanceType: "t2.nano", diff --git a/pkg/eks/client.go b/pkg/eks/client.go index fed5822258..1f8ae0252b 100644 --- a/pkg/eks/client.go +++ b/pkg/eks/client.go @@ -155,7 +155,7 @@ func (c *KubernetesProvider) WaitForControlPlane(meta *api.ClusterMeta, clientSe } // UpdateAuthConfigMap creates or adds a nodegroup IAM role in the auth ConfigMap for the given nodegroup. -func UpdateAuthConfigMap(ctx context.Context, nodeGroups []*api.NodeGroup, clientSet kubernetes.Interface) error { +func UpdateAuthConfigMap(nodeGroups []*api.NodeGroup, clientSet kubernetes.Interface) error { for _, ng := range nodeGroups { // skip ocean cluster if ng.SpotOcean != nil && ng.Name == api.SpotOceanClusterNodeGroupName { @@ -166,13 +166,6 @@ func UpdateAuthConfigMap(ctx context.Context, nodeGroups []*api.NodeGroup, clien if err := authconfigmap.AddNodeGroup(clientSet, ng); err != nil { return err } - - // wait for nodes to join - if ng.SpotOcean == nil { - if err := WaitForNodes(ctx, clientSet, ng); err != nil { - return err - } - } } return nil } diff --git a/pkg/eks/eks_test.go b/pkg/eks/eks_test.go index 02a4d4d6b1..173a740325 100644 --- a/pkg/eks/eks_test.go +++ b/pkg/eks/eks_test.go @@ -110,7 +110,7 @@ var _ = Describe("EKS API wrapper", func() { } } return matches == len(expectedStatusFilter) - })).Return(&cfn.ListStacksOutput{}, nil) + }), mock.Anything).Return(&cfn.ListStacksOutput{}, nil) }) JustBeforeEach(func() { diff --git a/pkg/eks/nodegroup_service_test.go b/pkg/eks/nodegroup_service_test.go index eaed9b0b2c..33d08e72d8 100644 --- a/pkg/eks/nodegroup_service_test.go +++ b/pkg/eks/nodegroup_service_test.go @@ -326,13 +326,13 @@ func mockOutpostInstanceTypes(provider *mockprovider.MockProvider) { instanceTypes[i] = it.InstanceType } - provider.MockOutposts().On("GetOutpostInstanceTypes", mock.Anything, mock.Anything).Return(&awsoutposts.GetOutpostInstanceTypesOutput{ + provider.MockOutposts().On("GetOutpostInstanceTypes", mock.Anything, mock.Anything, mock.Anything).Return(&awsoutposts.GetOutpostInstanceTypesOutput{ InstanceTypes: instanceTypeItems, }, nil) provider.MockEC2().On("DescribeInstanceTypes", mock.Anything, &ec2.DescribeInstanceTypesInput{ InstanceTypes: instanceTypes, - }).Return(&ec2.DescribeInstanceTypesOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: instanceTypeInfoList, }, nil) } diff --git a/pkg/eks/retryer_v2.go b/pkg/eks/retryer_v2.go index b491bb1103..30bf2dd950 100644 --- a/pkg/eks/retryer_v2.go +++ b/pkg/eks/retryer_v2.go @@ -1,15 +1,14 @@ package eks import ( + "errors" "net/http" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/smithy-go" - "github.com/pkg/errors" ) const ( @@ -39,7 +38,10 @@ func (r *RetryerV2) IsErrorRetryable(err error) bool { } var oe *smithy.OperationError - return errors.As(err, &oe) && oe.Err != nil && isErrorRetryable(oe.Err) + if !errors.As(err, &oe) { + return true + } + return oe.Err != nil && isErrorRetryable(oe.Err) } func isErrorRetryable(err error) bool { diff --git a/pkg/eks/services_v2.go b/pkg/eks/services_v2.go index 26154f6c1f..573a75417f 100644 --- a/pkg/eks/services_v2.go +++ b/pkg/eks/services_v2.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/ratelimit" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -85,6 +86,7 @@ func (s *ServicesV2) CloudFormation() awsapi.CloudFormation { o.StandardOptions = []func(*retry.StandardOptions){ func(so *retry.StandardOptions) { so.MaxAttempts = maxRetries + so.RateLimiter = ratelimit.None }, } }) diff --git a/pkg/eks/tasks.go b/pkg/eks/tasks.go index 3d9cc5eb4e..c17e28672e 100644 --- a/pkg/eks/tasks.go +++ b/pkg/eks/tasks.go @@ -28,14 +28,13 @@ import ( "github.com/weaveworks/eksctl/pkg/actions/irsa" "github.com/weaveworks/eksctl/pkg/addons" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/manager" "github.com/weaveworks/eksctl/pkg/fargate" iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" + "github.com/weaveworks/eksctl/pkg/kubernetes" instanceutils "github.com/weaveworks/eksctl/pkg/utils/instance" "github.com/weaveworks/eksctl/pkg/utils/tasks" - - api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" - "github.com/weaveworks/eksctl/pkg/kubernetes" ) type clusterConfigTask struct { @@ -279,10 +278,11 @@ func (t *restartDaemonsetTask) Do(errCh chan error) error { } // CreateExtraClusterConfigTasks returns all tasks for updating cluster configuration -func (c *ClusterProvider) CreateExtraClusterConfigTasks(ctx context.Context, cfg *api.ClusterConfig) *tasks.TaskTree { +func (c *ClusterProvider) CreateExtraClusterConfigTasks(ctx context.Context, cfg *api.ClusterConfig, preNodeGroupAddons *tasks.TaskTree, updateVPCCNITask *tasks.GenericTask) *tasks.TaskTree { newTasks := &tasks.TaskTree{ Parallel: false, IsSubTask: true, + Tasks: []tasks.Task{preNodeGroupAddons}, } newTasks.Append(&tasks.GenericTask{ @@ -302,6 +302,13 @@ func (c *ClusterProvider) CreateExtraClusterConfigTasks(ctx context.Context, cfg }, }) + if api.IsEnabled(cfg.IAM.WithOIDC) { + c.appendCreateTasksForIAMServiceAccounts(ctx, cfg, newTasks) + if updateVPCCNITask != nil { + newTasks.Append(updateVPCCNITask) + } + } + if cfg.HasClusterCloudWatchLogging() { if logRetentionDays := cfg.CloudWatch.ClusterLogging.LogRetentionInDays; logRetentionDays != 0 { newTasks.Append(&clusterConfigTask{ @@ -334,10 +341,6 @@ func (c *ClusterProvider) CreateExtraClusterConfigTasks(ctx context.Context, cfg }) } - if api.IsEnabled(cfg.IAM.WithOIDC) { - c.appendCreateTasksForIAMServiceAccounts(ctx, cfg, newTasks) - } - if len(cfg.IdentityProviders) > 0 { newTasks.Append(identityproviders.NewAssociateProvidersTask(ctx, *cfg.Metadata, cfg.IdentityProviders, c.AWSProvider.EKS())) } @@ -525,16 +528,10 @@ func (c *ClusterProvider) appendCreateTasksForIAMServiceAccounts(ctx context.Con // given a clientSet getter and OpenIDConnectManager reference we can build out // the list of tasks for each of the service accounts that need to be created newTasks := c.NewStackManager(cfg).NewTasksToCreateIAMServiceAccounts( - api.IAMServiceAccountsWithImplicitServiceAccounts(cfg), + cfg.IAM.ServiceAccounts, oidcPlaceholder, clientSet, ) newTasks.IsSubTask = true tasks.Append(newTasks) - tasks.Append(&restartDaemonsetTask{ - namespace: "kube-system", - name: "aws-node", - clusterProvider: c, - spec: cfg, - }) } diff --git a/pkg/nodebootstrap/al2023.go b/pkg/nodebootstrap/al2023.go index be587be681..1943f92823 100644 --- a/pkg/nodebootstrap/al2023.go +++ b/pkg/nodebootstrap/al2023.go @@ -49,6 +49,7 @@ func newAL2023Bootstrapper(cfg *api.ClusterConfig, np api.NodePool, clusterDNS s cfg: cfg, nodePool: np, clusterDNS: clusterDNS, + scripts: []string{assets.AL2023XTablesLock}, } } diff --git a/pkg/nodebootstrap/al2023_test.go b/pkg/nodebootstrap/al2023_test.go index 5b62bffe9e..56568adaaa 100644 --- a/pkg/nodebootstrap/al2023_test.go +++ b/pkg/nodebootstrap/al2023_test.go @@ -48,13 +48,13 @@ var _ = DescribeTable("Unmanaged AL2023", func(e al2023Entry) { Expect(actual).To(Equal(e.expectedUserData)) }, Entry("default", al2023Entry{ - expectedUserData: wrapMIMEParts(nodeConfig), + expectedUserData: wrapMIMEParts(xTablesLock + nodeConfig), }), Entry("efa enabled", al2023Entry{ overrideNodegroupSettings: func(np api.NodePool) { np.BaseNodeGroup().EFAEnabled = aws.Bool(true) }, - expectedUserData: wrapMIMEParts(efaScript + nodeConfig), + expectedUserData: wrapMIMEParts(xTablesLock + efaScript + nodeConfig), }), ) @@ -83,26 +83,26 @@ var _ = DescribeTable("Managed AL2023", func(e al2023Entry) { Expect(actual).To(Equal(e.expectedUserData)) }, Entry("native AMI", al2023Entry{ - expectedUserData: "", + expectedUserData: wrapMIMEParts(xTablesLock), }), Entry("native AMI && EFA enabled", al2023Entry{ overrideNodegroupSettings: func(np api.NodePool) { np.BaseNodeGroup().EFAEnabled = aws.Bool(true) }, - expectedUserData: wrapMIMEParts(efaCloudhook), + expectedUserData: wrapMIMEParts(xTablesLock + efaCloudhook), }), Entry("custom AMI", al2023Entry{ overrideNodegroupSettings: func(np api.NodePool) { np.BaseNodeGroup().AMI = "ami-xxxx" }, - expectedUserData: wrapMIMEParts(managedNodeConfig), + expectedUserData: wrapMIMEParts(xTablesLock + managedNodeConfig), }), Entry("custom AMI && EFA enabled", al2023Entry{ overrideNodegroupSettings: func(np api.NodePool) { np.BaseNodeGroup().AMI = "ami-xxxx" np.BaseNodeGroup().EFAEnabled = aws.Bool(true) }, - expectedUserData: wrapMIMEParts(efaCloudhook + managedNodeConfig), + expectedUserData: wrapMIMEParts(xTablesLock + efaCloudhook + managedNodeConfig), }), ) @@ -274,6 +274,13 @@ Content-Type: multipart/mixed; boundary=// ` } + xTablesLock = fmt.Sprintf(`--// +Content-Type: text/x-shellscript +Content-Type: charset="us-ascii" + +%s +`, assets.AL2023XTablesLock) + efaCloudhook = fmt.Sprintf(`--// Content-Type: text/cloud-boothook Content-Type: charset="us-ascii" diff --git a/pkg/nodebootstrap/assets/assets.go b/pkg/nodebootstrap/assets/assets.go index c77c5bc450..94fdc480b2 100644 --- a/pkg/nodebootstrap/assets/assets.go +++ b/pkg/nodebootstrap/assets/assets.go @@ -20,6 +20,11 @@ var BootstrapHelperSh string //go:embed scripts/bootstrap.ubuntu.sh var BootstrapUbuntuSh string +// AL2023XTablesLock holds the contents for creating a lock file for AL2023 AMIs. +// +//go:embed scripts/al2023-xtables.lock.sh +var AL2023XTablesLock string + // EfaAl2Sh holds the efa.al2.sh contents // //go:embed scripts/efa.al2.sh diff --git a/pkg/nodebootstrap/assets/scripts/al2023-xtables.lock.sh b/pkg/nodebootstrap/assets/scripts/al2023-xtables.lock.sh new file mode 100644 index 0000000000..5fa346b947 --- /dev/null +++ b/pkg/nodebootstrap/assets/scripts/al2023-xtables.lock.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +touch /run/xtables.lock diff --git a/pkg/nodebootstrap/assets/scripts/efa.al2.sh b/pkg/nodebootstrap/assets/scripts/efa.al2.sh index 8179c983af..f99ae6b962 100644 --- a/pkg/nodebootstrap/assets/scripts/efa.al2.sh +++ b/pkg/nodebootstrap/assets/scripts/efa.al2.sh @@ -7,6 +7,7 @@ set -o nounset yum install -y wget wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer.tar.gz tar -xf /tmp/aws-efa-installer.tar.gz -C /tmp +rm -rf /tmp/aws-efa-installer.tar.gz cd /tmp/aws-efa-installer ./efa_installer.sh -y -g /opt/amazon/efa/bin/fi_info -p efa diff --git a/pkg/nodebootstrap/assets/scripts/efa.al2023.sh b/pkg/nodebootstrap/assets/scripts/efa.al2023.sh index 3aef0ce36f..b73f630813 100644 --- a/pkg/nodebootstrap/assets/scripts/efa.al2023.sh +++ b/pkg/nodebootstrap/assets/scripts/efa.al2023.sh @@ -7,6 +7,7 @@ set -o nounset dnf install -y wget wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer.tar.gz tar -xf /tmp/aws-efa-installer.tar.gz -C /tmp +rm -rf /tmp/aws-efa-installer.tar.gz cd /tmp/aws-efa-installer ./efa_installer.sh -y -g /opt/amazon/efa/bin/fi_info -p efa diff --git a/pkg/nodebootstrap/assets/scripts/efa.managed.al2023.boothook b/pkg/nodebootstrap/assets/scripts/efa.managed.al2023.boothook index 5d2a081688..d8440a4520 100644 --- a/pkg/nodebootstrap/assets/scripts/efa.managed.al2023.boothook +++ b/pkg/nodebootstrap/assets/scripts/efa.managed.al2023.boothook @@ -2,6 +2,7 @@ cloud-init-per once dnf_wget dnf install -y wget cloud-init-per once wget_efa wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer-latest.tar.gz cloud-init-per once tar_efa tar -xf /tmp/aws-efa-installer-latest.tar.gz -C /tmp +cloud-init-per once rm_efa_gz rm -rf /tmp/aws-efa-installer-latest.tar.gz pushd /tmp/aws-efa-installer cloud-init-per once install_efa ./efa_installer.sh -y -g pop /tmp/aws-efa-installer diff --git a/pkg/nodebootstrap/assets/scripts/efa.managed.boothook b/pkg/nodebootstrap/assets/scripts/efa.managed.boothook index 32e191cd24..d2863d42c6 100644 --- a/pkg/nodebootstrap/assets/scripts/efa.managed.boothook +++ b/pkg/nodebootstrap/assets/scripts/efa.managed.boothook @@ -2,6 +2,7 @@ cloud-init-per once yum_wget yum install -y wget cloud-init-per once wget_efa wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer-latest.tar.gz cloud-init-per once tar_efa tar -xf /tmp/aws-efa-installer-latest.tar.gz -C /tmp +cloud-init-per once rm_efa_gz rm -rf /tmp/aws-efa-installer-latest.tar.gz pushd /tmp/aws-efa-installer cloud-init-per once install_efa ./efa_installer.sh -y -g pop /tmp/aws-efa-installer diff --git a/pkg/nodebootstrap/managed_al2_test.go b/pkg/nodebootstrap/managed_al2_test.go index d463eea253..6b9c08dcd4 100644 --- a/pkg/nodebootstrap/managed_al2_test.go +++ b/pkg/nodebootstrap/managed_al2_test.go @@ -111,6 +111,7 @@ cloud-init-per once yum_wget yum install -y wget cloud-init-per once wget_efa wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer-latest.tar.gz cloud-init-per once tar_efa tar -xf /tmp/aws-efa-installer-latest.tar.gz -C /tmp +cloud-init-per once rm_efa_gz rm -rf /tmp/aws-efa-installer-latest.tar.gz pushd /tmp/aws-efa-installer cloud-init-per once install_efa ./efa_installer.sh -y -g pop /tmp/aws-efa-installer @@ -143,6 +144,7 @@ cloud-init-per once yum_wget yum install -y wget cloud-init-per once wget_efa wget -q --timeout=20 https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz -O /tmp/aws-efa-installer-latest.tar.gz cloud-init-per once tar_efa tar -xf /tmp/aws-efa-installer-latest.tar.gz -C /tmp +cloud-init-per once rm_efa_gz rm -rf /tmp/aws-efa-installer-latest.tar.gz pushd /tmp/aws-efa-installer cloud-init-per once install_efa ./efa_installer.sh -y -g pop /tmp/aws-efa-installer diff --git a/pkg/outposts/cluster_extender_test.go b/pkg/outposts/cluster_extender_test.go index 33d416069f..045e64fef5 100644 --- a/pkg/outposts/cluster_extender_test.go +++ b/pkg/outposts/cluster_extender_test.go @@ -728,7 +728,7 @@ func mockDescribeSubnets(provider *mockprovider.MockProvider, clusterSubnets *ap Values: []string{"vpc-1234"}, }, }, - }).Return(&ec2.DescribeSubnetsOutput{ + }, mock.Anything).Return(&ec2.DescribeSubnetsOutput{ Subnets: allSubnets, }, nil) } diff --git a/pkg/outposts/outposts_test.go b/pkg/outposts/outposts_test.go index e72cc368d4..090bfc6231 100644 --- a/pkg/outposts/outposts_test.go +++ b/pkg/outposts/outposts_test.go @@ -229,13 +229,13 @@ func mockOutpostInstanceTypes(provider *mockprovider.MockProvider) { } instanceTypes[i] = it.InstanceType } - provider.MockOutposts().On("GetOutpostInstanceTypes", mock.Anything, mock.Anything).Return(&awsoutposts.GetOutpostInstanceTypesOutput{ + provider.MockOutposts().On("GetOutpostInstanceTypes", mock.Anything, mock.Anything, mock.Anything).Return(&awsoutposts.GetOutpostInstanceTypesOutput{ InstanceTypes: instanceTypeItems, }, nil) provider.MockEC2().On("DescribeInstanceTypes", mock.Anything, &ec2.DescribeInstanceTypesInput{ InstanceTypes: instanceTypes, - }).Return(&ec2.DescribeInstanceTypesOutput{ + }, mock.Anything).Return(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: instanceTypeInfoList, }, nil) } diff --git a/pkg/printers/testdata/jsontest_2clusters.golden b/pkg/printers/testdata/jsontest_2clusters.golden index 4c919cf353..7845b1206a 100644 --- a/pkg/printers/testdata/jsontest_2clusters.golden +++ b/pkg/printers/testdata/jsontest_2clusters.golden @@ -35,7 +35,9 @@ "RoleArn": null, "Status": "ACTIVE", "Tags": null, - "Version": null + "Version": null, + "UpgradePolicy": null, + "ZonalShiftConfig": null }, { "Id": null, @@ -73,6 +75,8 @@ "RoleArn": null, "Status": "ACTIVE", "Tags": null, - "Version": null + "Version": null, + "UpgradePolicy": null, + "ZonalShiftConfig": null } ] diff --git a/pkg/printers/testdata/jsontest_single.golden b/pkg/printers/testdata/jsontest_single.golden index 1c8e2d653a..8403c144dc 100644 --- a/pkg/printers/testdata/jsontest_single.golden +++ b/pkg/printers/testdata/jsontest_single.golden @@ -34,6 +34,8 @@ "RoleArn": null, "Status": "ACTIVE", "Tags": null, - "Version": null + "Version": null, + "UpgradePolicy": null, + "ZonalShiftConfig": null } ] diff --git a/pkg/printers/testdata/yamltest_2clusters.golden b/pkg/printers/testdata/yamltest_2clusters.golden index 60e61d8f3a..17306cded2 100644 --- a/pkg/printers/testdata/yamltest_2clusters.golden +++ b/pkg/printers/testdata/yamltest_2clusters.golden @@ -30,6 +30,8 @@ Status: ACTIVE Tags: null Version: null + UpgradePolicy: null + ZonalShiftConfig: null - Id: null Arn: arn-87654321 CertificateAuthority: null @@ -62,3 +64,5 @@ Status: ACTIVE Tags: null Version: null + UpgradePolicy: null + ZonalShiftConfig: null diff --git a/pkg/printers/testdata/yamltest_single.golden b/pkg/printers/testdata/yamltest_single.golden index 9c99e5aca2..55026b7e2f 100644 --- a/pkg/printers/testdata/yamltest_single.golden +++ b/pkg/printers/testdata/yamltest_single.golden @@ -30,3 +30,5 @@ Status: ACTIVE Tags: null Version: null + UpgradePolicy: null + ZonalShiftConfig: null diff --git a/pkg/spot/types.go b/pkg/spot/types.go index 9dbdcff81a..faddef9351 100644 --- a/pkg/spot/types.go +++ b/pkg/spot/types.go @@ -155,6 +155,7 @@ type ( ResourceLimits *ResourceLimits `json:"resourceLimits,omitempty"` Headroom *Headroom `json:"headroom,omitempty"` // cluster Headrooms []*Headroom `json:"headrooms,omitempty"` // virtualnodegroup + Down *AutoScalerDown `json:"down,omitempty"` } Headroom struct { @@ -171,6 +172,16 @@ type ( MaxInstanceCount *int `json:"maxInstanceCount,omitempty"` } + AutoScalerDown struct { + EvaluationPeriods *int `json:"evaluationPeriods,omitempty"` + MaxScaleDownPercentage *float64 `json:"maxScaleDownPercentage,omitempty"` + AggressiveScaleDown *AggressiveScaleDown `json:"aggressiveScaleDown,omitempty"` + } + + AggressiveScaleDown struct { + IsEnabled *bool `json:"isEnabled,omitempty"` + } + Label struct { Key *string `json:"key,omitempty"` Value *string `json:"value,omitempty"` diff --git a/pkg/utils/instance/instance.go b/pkg/utils/instance/instance.go index 024012bb3a..211c668702 100644 --- a/pkg/utils/instance/instance.go +++ b/pkg/utils/instance/instance.go @@ -13,6 +13,7 @@ func IsARMInstanceType(instanceType string) bool { strings.HasPrefix(instanceType, "t4g") || strings.HasPrefix(instanceType, "m6g") || strings.HasPrefix(instanceType, "m7g") || + strings.HasPrefix(instanceType, "m8g") || strings.HasPrefix(instanceType, "c6g") || strings.HasPrefix(instanceType, "c7g") || strings.HasPrefix(instanceType, "r6g") || @@ -20,6 +21,7 @@ func IsARMInstanceType(instanceType string) bool { strings.HasPrefix(instanceType, "im4g") || strings.HasPrefix(instanceType, "is4g") || strings.HasPrefix(instanceType, "g5g") || + strings.HasPrefix(instanceType, "hpc7g") || strings.HasPrefix(instanceType, "x2g") } diff --git a/pkg/utils/tasks/tasks.go b/pkg/utils/tasks/tasks.go index 6ccb0befde..a7fc346eb1 100644 --- a/pkg/utils/tasks/tasks.go +++ b/pkg/utils/tasks/tasks.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/kris-nova/logger" + "golang.org/x/sync/errgroup" ) // Task is a common interface for the stack manager tasks. @@ -50,6 +51,7 @@ type TaskTree struct { Parallel bool PlanMode bool IsSubTask bool + Limit int } // Append new tasks to the set @@ -147,7 +149,11 @@ func (t *TaskTree) Do(allErrs chan error) error { errs := make(chan error) if t.Parallel { - go doParallelTasks(errs, t.Tasks) + if t.Limit > 0 { + go runInErrorGroup(t.Tasks, t.Limit, errs) + } else { + go doParallelTasks(errs, t.Tasks) + } } else { go doSequentialTasks(errs, t.Tasks) } @@ -173,7 +179,11 @@ func (t *TaskTree) DoAllSync() []error { errs := make(chan error) if t.Parallel { - go doParallelTasks(errs, t.Tasks) + if t.Limit > 0 { + go runInErrorGroup(t.Tasks, t.Limit, errs) + } else { + go doParallelTasks(errs, t.Tasks) + } } else { go doSequentialTasks(errs, t.Tasks) } @@ -217,6 +227,24 @@ func doParallelTasks(allErrs chan error, tasks []Task) { close(allErrs) } +func runInErrorGroup(tasks []Task, limit int, errs chan error) { + var eg errgroup.Group + eg.SetLimit(limit) + for _, t := range tasks { + t := t + eg.Go(func() error { + if ok := doSingleTask(errs, t); !ok { + logger.Debug("failed task: %s (will continue until other parallel tasks are completed)", t.Describe()) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + logger.Debug("error running tasks: %v", err) + } + close(errs) +} + func doSequentialTasks(allErrs chan error, tasks []Task) { for t := range tasks { if ok := doSingleTask(allErrs, tasks[t]); !ok { diff --git a/pkg/version/release.go b/pkg/version/release.go index 40045543f1..0e5faf4cf5 100644 --- a/pkg/version/release.go +++ b/pkg/version/release.go @@ -3,7 +3,7 @@ package version // This file was generated by release_generate.go; DO NOT EDIT. // Version is the version number in semver format X.Y.Z -var Version = "0.183.0" +var Version = "0.194.0" // PreReleaseID can be empty for releases, "rc.X" for release candidates and "dev" for snapshots var PreReleaseID = "dev" diff --git a/pkg/vpc/vpc_test.go b/pkg/vpc/vpc_test.go index e4090852e4..c4c9ef8985 100644 --- a/pkg/vpc/vpc_test.go +++ b/pkg/vpc/vpc_test.go @@ -499,7 +499,7 @@ var _ = Describe("VPC", func() { }, mockEC2: func(ec2Mock *mocksv2.EC2) { - ec2Mock.On("DescribeSubnets", Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { + ec2Mock.On("DescribeSubnets", Anything, Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { if len(input.Filters) > 0 { return &ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ @@ -590,7 +590,7 @@ var _ = Describe("VPC", func() { }, mockEC2: func(ec2Mock *mocksv2.EC2) { - ec2Mock.On("DescribeSubnets", Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { + ec2Mock.On("DescribeSubnets", Anything, Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { if len(input.Filters) > 0 { return &ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ @@ -673,7 +673,7 @@ var _ = Describe("VPC", func() { }, }, mockEC2: func(ec2Mock *mocksv2.EC2) { - ec2Mock.On("DescribeSubnets", Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { + ec2Mock.On("DescribeSubnets", Anything, Anything, Anything).Return(func(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(options *ec2.Options)) *ec2.DescribeSubnetsOutput { if len(input.Filters) > 0 { return &ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ @@ -1218,6 +1218,7 @@ var _ = Describe("VPC", func() { }, { Name: strings.Pointer("cidr-block"), Values: []string{"192.168.64.0/18"}, }}}, + Anything, ).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { @@ -1235,6 +1236,7 @@ var _ = Describe("VPC", func() { }, { Name: strings.Pointer("availability-zone"), Values: []string{"az3"}, }}}, + Anything, ).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { @@ -1248,6 +1250,7 @@ var _ = Describe("VPC", func() { p.MockEC2().On("DescribeSubnets", Anything, &ec2.DescribeSubnetsInput{SubnetIds: []string{"private1"}}, + Anything, ).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { @@ -1262,6 +1265,7 @@ var _ = Describe("VPC", func() { p.MockEC2().On("DescribeSubnets", Anything, &ec2.DescribeSubnetsInput{SubnetIds: []string{"public1"}}, + Anything, ).Return(&ec2.DescribeSubnetsOutput{ Subnets: []ec2types.Subnet{ { diff --git a/userdocs/requirements.txt b/userdocs/requirements.txt index 8e30421a4d..46363cfdf2 100644 --- a/userdocs/requirements.txt +++ b/userdocs/requirements.txt @@ -4,11 +4,11 @@ mkdocs-redirects mkdocs-minify-plugin mkdocs-glightbox pymdown-extensions >= 9.9.1 -jinja2 == 3.1.3 -pillow +jinja2 == 3.1.4 +pillow cairosvg -# Dependencies from material theme +# Dependencies from material theme mkdocs-material-extensions>=1.1 pygments>=2.12 -markdown>=3.2 \ No newline at end of file +markdown>=3.2 diff --git a/userdocs/src/getting-started.md b/userdocs/src/getting-started.md index a41463088d..5bd302c39c 100644 --- a/userdocs/src/getting-started.md +++ b/userdocs/src/getting-started.md @@ -1,6 +1,8 @@ # Getting started !!! tip "New for 2024" + `eksctl` now supports new region Kuala Lumpur (`ap-southeast-5`) + EKS Add-ons now support receiving IAM permissions via [EKS Pod Identity Associations](/usage/pod-identity-associations/#eks-add-ons-support-for-pod-identity-associations) `eksctl` now supports AMIs based on AmazonLinux2023 @@ -122,7 +124,7 @@ eksctl create cluster --name=cluster-1 --nodes=4 ### Supported versions -EKS supports versions `1.23` (extended), `1.24` (extended), `1.25`, `1.26`, `1.27`, `1.28`, `1.29` and **`1.30`** (default). +EKS supports versions `1.23` (extended), `1.24` (extended), `1.25`, `1.26`, `1.27`, `1.28`, `1.29`, **`1.30`** (default) and `1.31`. With `eksctl` you can deploy any of the supported versions by passing `--version`. ```sh diff --git a/userdocs/src/usage/addon-upgrade.md b/userdocs/src/usage/addon-upgrade.md index e8899cac41..e7fec55176 100644 --- a/userdocs/src/usage/addon-upgrade.md +++ b/userdocs/src/usage/addon-upgrade.md @@ -1,5 +1,12 @@ # Default add-on updates +!!! warning "New for 2024" + eksctl now installs default addons as EKS addons instead of self-managed addons. Read more about its implications in [Cluster creation flexibility for default networking addons](#cluster-creation-flexibility-for-default-networking-addons). + +!!! warning "New for 2024" + For updating addons, `eksctl utils update-*` cannot be used for clusters created with eksctl v0.184.0 and above. + This guide is only valid for clusters created before this change. + There are 3 default add-ons that get included in each EKS cluster: - `kube-proxy` - `aws-node` diff --git a/userdocs/src/usage/addons.md b/userdocs/src/usage/addons.md index 05600fa74b..236d6dd884 100644 --- a/userdocs/src/usage/addons.md +++ b/userdocs/src/usage/addons.md @@ -6,6 +6,12 @@ CNI plugin through the EKS API ## Creating addons (and providing IAM permissions via IRSA) +!!! tip "New for 2024" + eksctl now supports creating clusters without any default networking addons: [Cluster creation flexibility for default networking addons](#cluster-creation-flexibility-for-default-networking-addons). + +!!! warning "New for 2024" + eksctl now installs default addons as EKS addons instead of self-managed addons. Read more about its implications in [Cluster creation flexibility for default networking addons](#cluster-creation-flexibility-for-default-networking-addons). + !!! tip "New for 2024" EKS Add-ons now support receiving IAM permissions, required to connect with AWS services outside of cluster, via [EKS Pod Identity Associations](/usage/pod-identity-associations/#eks-add-ons-support-for-pod-identity-associations) @@ -87,8 +93,8 @@ addons: For addon create, the `resolveConflicts` field supports three distinct values: -- `none` - EKS doesn't change the value. The create might fail. -- `overwrite` - EKS overwrites any config changes back to EKS default values. +- `none` - EKS doesn't change the value. The create might fail. +- `overwrite` - EKS overwrites any config changes back to EKS default values. - `preserve` - EKS doesn't change the value. The create might fail. (Similarly to `none`, but different from [`preserve` in updating addons](#updating-addons)) ## Listing enabled addons @@ -141,7 +147,7 @@ eksctl utils describe-addon-configuration --name vpc-cni --version v1.12.0-eksbu This returns a JSON schema of the various options available for this addon. ## Working with configuration values -`ConfigurationValues` can be provided in the configuration file during the creation or update of addons. Only JSON and YAML formats are supported. +`ConfigurationValues` can be provided in the configuration file during the creation or update of addons. Only JSON and YAML formats are supported. For eg., @@ -202,10 +208,10 @@ addons: resolveConflicts: preserve ``` -For addon update, the `resolveConflicts` field accepts three distinct values: +For addon update, the `resolveConflicts` field accepts three distinct values: - `none` - EKS doesn't change the value. The update might fail. -- `overwrite` - EKS overwrites any config changes back to EKS default values. +- `overwrite` - EKS overwrites any config changes back to EKS default values. - `preserve` - EKS preserves the value. If you choose this option, we recommend that you test any field and value changes on a non-production cluster before updating the add-on on your production cluster. ## Deleting addons @@ -216,3 +222,48 @@ eksctl delete addon --cluster --name This will delete the addon and any IAM roles associated to it. When you delete your cluster all IAM roles associated to addons are also deleted. + +## Cluster creation flexibility for default networking addons + +When a cluster is created, EKS automatically installs VPC CNI, CoreDNS and kube-proxy as self-managed addons. +To disable this behavior in order to use other CNI plugins like Cilium and Calico, eksctl now supports creating a cluster +without any default networking addons. To create such a cluster, set `addonsConfig.disableDefaultAddons`, as in: + +```yaml +addonsConfig: + disableDefaultAddons: true +``` + +```shell +$ eksctl create cluster -f cluster.yaml +``` + +To create a cluster with only CoreDNS and kube-proxy and not VPC CNI, specify the addons explicitly in `addons` +and set `addonsConfig.disableDefaultAddons`, as in: + +```yaml +addonsConfig: + disableDefaultAddons: true +addons: + - name: kube-proxy + - name: coredns +``` + +```shell +$ eksctl create cluster -f cluster.yaml +``` + +As part of this change, eksctl now installs default addons as EKS addons instead of self-managed addons during cluster creation +if `addonsConfig.disableDefaultAddons` is not explicitly set to true. As such, `eksctl utils update-*` commands can no +longer be used for updating addons for clusters created with eksctl v0.184.0 and above: + +- `eksctl utils update-aws-node` +- `eksctl utils update-coredns` +- `eksctl utils update-kube-proxy` + +Instead, `eksctl update addon` should be used now. + +To learn more, see [EKS documentation][eksdocs]. + + +[eksdocs]: https://aws.amazon.com/about-aws/whats-new/2024/06/amazon-eks-cluster-creation-flexibility-networking-add-ons/ diff --git a/userdocs/src/usage/cluster-upgrade.md b/userdocs/src/usage/cluster-upgrade.md index 26dd490eb1..762343cdf1 100644 --- a/userdocs/src/usage/cluster-upgrade.md +++ b/userdocs/src/usage/cluster-upgrade.md @@ -12,11 +12,11 @@ An _`eksctl`-managed_ cluster can be upgraded in 3 easy steps: Please make sure to read this section in full before you proceed. ???+ info - Kubernetes supports version drift of up-to two minor versions during upgrade - process. So nodes can be up to two minor versions ahead or behind the control plane + Kubernetes supports version drift of up to two minor versions during the upgrade + process. Nodes can be up to two minor versions behind, but never ahead of the control plane version. You can only upgrade the control plane one minor version at a time, but - nodes can be upgraded more than one minor version at a time, provided the nodes stay - within two minor versions of the control plane. + nodes can be upgraded more than one minor version at a time, provided their version + does not become greater than the control plane version. ???+ info The old `eksctl update cluster` will be deprecated. Use `eksctl upgrade cluster` instead. diff --git a/userdocs/src/usage/gpu-support.md b/userdocs/src/usage/gpu-support.md index 07d95d60c5..5fe1a41760 100644 --- a/userdocs/src/usage/gpu-support.md +++ b/userdocs/src/usage/gpu-support.md @@ -24,7 +24,17 @@ use `--install-nvidia-plugin=false` with the create command. For example: ``` eksctl create cluster --node-type=p2.xlarge --install-nvidia-plugin=false +``` + +and, for versions 0.15.0 and above, +``` +kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin//deployments/static/nvidia-device-plugin.yml +``` + +or, for older versions, + +``` kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin//nvidia-device-plugin.yml ``` diff --git a/userdocs/src/usage/zonal-shift.md b/userdocs/src/usage/zonal-shift.md new file mode 100644 index 0000000000..af2f5ef88c --- /dev/null +++ b/userdocs/src/usage/zonal-shift.md @@ -0,0 +1,48 @@ +# Support for Zonal Shift in EKS clusters + +EKS now supports Amazon Application Recovery Controller (ARC) zonal shift and zonal autoshift that enhances the +resiliency of multi-AZ cluster environments. With AWS Zonal Shift, customers can shift in-cluster traffic away +from an impaired availability zone, ensuring new Kubernetes pods and nodes are launched in healthy availability zones only. + +## Creating a cluster with zonal shift enabled + +```yaml +# zonal-shift-cluster.yaml +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: highly-available-cluster + region: us-west-2 + + +zonalShiftConfig: + enabled: true + +``` + +```shell +$ eksctl create cluster -f zonal-shift-cluster.yaml +``` + + +## Enabling zonal shift on an existing cluster + +To enable or disable zonal shift on an existing cluster, run + +```shell +$ eksctl utils update-zonal-shift-config -f zonal-shift-cluster.yaml +``` + +or without a config file: + +```shell +$ eksctl utils update-zonal-shift-config --cluster=zonal-shift-cluster --enabled +``` + +## Further information + +- [EKS Zonal Shift][eks-user-guide] + +[eks-user-guide]: https://docs.aws.amazon.com/eks/latest/userguide/zone-shift.html diff --git a/userdocs/theme/home.html b/userdocs/theme/home.html index d8955c760b..8fb824af8b 100644 --- a/userdocs/theme/home.html +++ b/userdocs/theme/home.html @@ -533,6 +533,7 @@

eksctl create cluster

Usage Outposts

Check out latest eksctl features

+

Support for Kuala Lumpur region (ap-southeast-5)

EKS Add-ons support receiving IAM permissions via EKS Pod Identity Associations.

Support for AMIs based on AmazonLinux2023

Configuring cluster access management via AWS EKS Access Entries.

diff --git a/userdocs/theme/main.html b/userdocs/theme/main.html index f0d46da3eb..33306cf7c8 100644 --- a/userdocs/theme/main.html +++ b/userdocs/theme/main.html @@ -6,6 +6,13 @@ eksctl is now fully maintained by AWS. For more details check out eksctl Support Status Update.

+

+ eksctl now supports Cluster creation flexibility for networking add-ons. +

+

+ eksctl now installs default addons as EKS addons instead of self-managed addons. To understand its implications, check out + Cluster creation flexibility for networking add-ons. +

{% endblock %} @@ -13,4 +20,4 @@ {{ super() }} -{% endblock %} \ No newline at end of file +{% endblock %}