From 3d97c3d9f0f74cc9b08adb3a2a67edbf2276e6ff Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 18 Dec 2024 12:47:07 +0000 Subject: [PATCH] Add ValidatingAdmissionPolicy tests for RayClusters and AppWrappers --- go.mod | 31 +- go.sum | 71 ++--- tests/odh/support.go | 79 ++++++ tests/odh/validating_admission_policy_test.go | 268 ++++++++++++++++++ 4 files changed, 397 insertions(+), 52 deletions(-) create mode 100644 tests/odh/validating_admission_policy_test.go diff --git a/go.mod b/go.mod index 7d7f6251..fe7b2261 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,19 @@ module github.com/opendatahub-io/distributed-workloads -go 1.21 - -toolchain go1.21.5 +go 1.22.2 require ( github.com/kubeflow/training-operator v1.7.0 - github.com/onsi/gomega v1.31.1 + github.com/matoous/go-nanoid/v2 v2.1.0 + github.com/onsi/gomega v1.32.0 + github.com/project-codeflare/appwrapper v0.8.0 github.com/project-codeflare/codeflare-common v0.0.0-20241211130338-efe4f3e6f904 github.com/prometheus/client_golang v1.20.4 github.com/prometheus/common v0.57.0 github.com/ray-project/kuberay/ray-operator v1.1.0-alpha.0 - k8s.io/api v0.29.2 - k8s.io/apimachinery v0.29.2 + k8s.io/api v0.30.8 + k8s.io/apimachinery v0.30.8 + k8s.io/client-go v0.30.8 sigs.k8s.io/kueue v0.6.2 ) @@ -24,9 +25,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -34,7 +34,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -55,7 +55,6 @@ require ( github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec // indirect github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/project-codeflare/appwrapper v0.8.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -72,13 +71,11 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.0 // indirect - k8s.io/client-go v0.29.2 // indirect - k8s.io/component-base v0.29.1 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/controller-runtime v0.17.0 // indirect + sigs.k8s.io/controller-runtime v0.18.6 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 79ef93fc..bf9c29d2 100644 --- a/go.sum +++ b/go.sum @@ -88,10 +88,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -105,9 +105,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -162,12 +161,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= -github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -313,6 +312,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -343,14 +344,14 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= -github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec h1:rdkrEAVD8MeBimjIKkZ+wGm+TkTfG2eDEHUuAjAWkEg= @@ -845,22 +846,22 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g= -k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= -k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= -k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/api v0.30.8 h1:Y+yZRF3c1WC0MTkLe0qBkiLCquRNa4I21/iDioGMCbo= +k8s.io/api v0.30.8/go.mod h1:89IE5MzirZ5HHxU/Hq1/KWGqXkhXClu/FHGesFhQ0A4= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.8 h1:9jyTItYzmJc00cBDxZC5ArFNxUeKCwbw0m760iFUMKY= +k8s.io/apimachinery v0.30.8/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= +k8s.io/client-go v0.30.8 h1:fC1SQMZm7bSWiVv9ydN+nv+sqGVAxMdf/5eKUVffNJE= +k8s.io/client-go v0.30.8/go.mod h1:daF3UcGVqGPHvH5mn/ESkp/VoR8i9tg9IBfKr+AeFYo= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/metrics v0.29.1 h1:qutc3aIPMCniMuEApuLaeYX47rdCn8eycVDx7R6wMlQ= k8s.io/metrics v0.29.1/go.mod h1:JrbV2U71+v7d/9qb90UVKL8r0uJ6Z2Hy4V7mDm05cKs= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= @@ -868,10 +869,10 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= -sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= -sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= +sigs.k8s.io/controller-runtime v0.18.6 h1:UnEoLBLDpQwzJ2jYh6aTdiMhGjNDR7IdFn9YEqHIccc= +sigs.k8s.io/controller-runtime v0.18.6/go.mod h1:Dcsa9v8AEBWa3sQNJHsuWPT4ICv99irl5wj83NiC12U= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kueue v0.6.2 h1:wSX6DY/BNCIaza9R8TyhRxwzSrY02EkyxfXXeHbV758= diff --git a/tests/odh/support.go b/tests/odh/support.go index 0f7b6dc9..a1751b2c 100644 --- a/tests/odh/support.go +++ b/tests/odh/support.go @@ -21,11 +21,15 @@ import ( "net/url" "os" + gonanoid "github.com/matoous/go-nanoid/v2" gomega "github.com/onsi/gomega" + awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" "github.com/project-codeflare/codeflare-common/support" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) //go:embed resources/* @@ -65,3 +69,78 @@ func GetTestJobId(test support.Test, rayClient support.RayClusterClient) string return jobID } + +// Adds a unique suffix to the provided string +func uniqueSuffix(prefix string) string { + suffix := gonanoid.MustGenerate("1234567890abcdef", 4) + return prefix + "-" + suffix +} + +func newAppWrapperWithRayCluster(awName string, rcName string, namespace string) *awv1beta2.AppWrapper { + return &awv1beta2.AppWrapper{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "workload.codeflare.dev/v1beta2", + Kind: "AppWrapper", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: awName, + Namespace: namespace, + }, + Spec: awv1beta2.AppWrapperSpec{ + Components: []awv1beta2.AppWrapperComponent{ + { + Template: runtime.RawExtension{ + Object: &rayv1.RayCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "ray.io/v1", + Kind: "RayCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: rcName, + Namespace: namespace, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + RayStartParams: map[string]string{}, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "ray-head", + Image: "rayproject/ray:latest", + Ports: []v1.ContainerPort{ + { + Name: "redis", + ContainerPort: 6379, + }, + }, + }, + }, + }, + }, + }, + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ + { + GroupName: "workers", + Replicas: support.Ptr(int32(1)), + RayStartParams: map[string]string{}, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "ray-worker", + Image: "rayproject/ray:latest", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/tests/odh/validating_admission_policy_test.go b/tests/odh/validating_admission_policy_test.go new file mode 100644 index 00000000..49b4618c --- /dev/null +++ b/tests/odh/validating_admission_policy_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "encoding/json" + "testing" + "time" + + . "github.com/onsi/gomega" + awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + . "github.com/project-codeflare/codeflare-common/support" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + vapv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" +) + +// Note: This test must run on an OCP v4.17 or later cluster. +// The Validating Admission Policy feature gate is GA and enabled by default from OCP v4.17 (k8s v1.30) +func TestValidatingAdmissionPolicy(t *testing.T) { + test := With(t) + + var ( + ns *corev1.Namespace + rf *kueuev1beta1.ResourceFlavor + cq *kueuev1beta1.ClusterQueue + lq *kueuev1beta1.LocalQueue + rc *rayv1.RayCluster + aw *awv1beta2.AppWrapper + vapb *vapv1.ValidatingAdmissionPolicyBinding + vapbCopy *vapv1.ValidatingAdmissionPolicyBinding + awWithLQName = "aw-with-lq" + awNoLQName = "aw-no-lq" + rcWithLQName = "rc-with-lq" + rcNoLQName = "rc-no-lq" + ) + + // Register RayCluster types with the scheme + err := rayv1.AddToScheme(scheme.Scheme) + test.Expect(err).ToNot(HaveOccurred()) + + // Register AppWrapper types with the scheme + err = awv1beta2.AddToScheme(scheme.Scheme) + test.Expect(err).ToNot(HaveOccurred()) + + // Create a namespace + ns = CreateTestNamespaceWithName(test, uniqueSuffix("vap")) + defer test.Client().Core().CoreV1().Namespaces().Delete(test.Ctx(), ns.Name, metav1.DeleteOptions{}) + + // Create a resource flavor + rf = CreateKueueResourceFlavor(test, kueuev1beta1.ResourceFlavorSpec{}) + defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), rf.Name, metav1.DeleteOptions{}) + + // Create a cluster queue + cqSpec := kueuev1beta1.ClusterQueueSpec{ + NamespaceSelector: &metav1.LabelSelector{}, + ResourceGroups: []kueuev1beta1.ResourceGroup{ + { + CoveredResources: []corev1.ResourceName{corev1.ResourceName("cpu"), corev1.ResourceName("memory"), corev1.ResourceName("nvidia.com/gpu")}, + Flavors: []kueuev1beta1.FlavorQuotas{ + { + Name: kueuev1beta1.ResourceFlavorReference(rf.Name), + Resources: []kueuev1beta1.ResourceQuota{ + { + Name: corev1.ResourceCPU, + NominalQuota: resource.MustParse("3"), + }, + { + Name: corev1.ResourceMemory, + NominalQuota: resource.MustParse("8Gi"), + }, + { + Name: corev1.ResourceName("nvidia.com/gpu"), + NominalQuota: resource.MustParse("0"), + }, + }, + }, + }, + }, + }, + } + // Create a cluster queue + cq = CreateKueueClusterQueue(test, cqSpec) + defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), cq.Name, metav1.DeleteOptions{}) + + // Create a local queue + lq = CreateKueueLocalQueue(test, ns.Name, cq.Name) + defer test.Client().Kueue().KueueV1beta1().LocalQueues(ns.Name).Delete(test.Ctx(), lq.Name, metav1.DeleteOptions{}) + + // Snapshot the original ValidatingAdmissionPolicyBinding state + vapb, err = test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Get(test.Ctx(), "kueue-validating-admission-policy-binding", metav1.GetOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + + vapbCopy = vapb.DeepCopy() + defer revertVAPB(test, vapbCopy) + + /************************************************************************** + Testing the default behavior with the ValidatingAdmissionPolicyBinding enforcement enabled. + **************************************************************************/ + t.Run("RayCluster should be admitted with the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcWithLQName), ns.Name).Queue(lq.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("RayCluster should not be admitted without the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcNoLQName), ns.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(BeNil()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("AppWrapper should be admitted with the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + awName := uniqueSuffix(awWithLQName) + rcName := uniqueSuffix(rcNoLQName) + + aw = newAppWrapperWithRayCluster(awName, rcName, ns.Name) + if aw.Labels == nil { + aw.Labels = make(map[string]string) + } + aw.Labels["kueue.x-k8s.io/queue-name"] = lq.Name + + // Create the AppWrapper with the 'kueue.x-k8s.io/queue-name' label set + awMap, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + _, err = test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Create(test.Ctx(), &unstructured.Unstructured{Object: awMap}, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Delete(test.Ctx(), awName, metav1.DeleteOptions{}) + }) + t.Run("AppWrapper should be admitted without the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + aw = newAppWrapperWithRayCluster(uniqueSuffix(awNoLQName), uniqueSuffix(rcNoLQName), ns.Name) + + // Create the AppWrapper without the 'kueue.x-k8s.io/queue-name' label set + awMap, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + _, err = test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Create(test.Ctx(), &unstructured.Unstructured{Object: awMap}, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{}) + }) + + /************************************************************************** + Testing the 1st alternative behavior with the ValidatingAdmissionPolicyBinding enforcement disabled. + **************************************************************************/ + t.Run("Disable the ValidatingAdmissionPolicy enforcement", func(t *testing.T) { + vapb, err := test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Get(test.Ctx(), vapb.Name, metav1.GetOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + + vapb.Spec.PolicyName = "none" + _, err = test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Update(test.Ctx(), vapb, metav1.UpdateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + time.Sleep(2 * time.Second) // Wait for the ValidatingAdmissionPolicyBinding to be updated + defer revertVAPB(test, vapbCopy) + + t.Run("RayCluster should be admitted with the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcWithLQName), ns.Name).Queue(lq.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("RayCluster should be admitted without the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcNoLQName), ns.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("AppWrapper should be admitted with the 'kueue.x-k8s.io/queue-name' label set", func(t *testing.T) { + awName := uniqueSuffix(awWithLQName) + rcName := uniqueSuffix(rcNoLQName) + + aw = newAppWrapperWithRayCluster(awName, rcName, ns.Name) + if aw.Labels == nil { + aw.Labels = make(map[string]string) + } + aw.Labels["kueue.x-k8s.io/queue-name"] = lq.Name + + // Create the AppWrapper with the 'kueue.x-k8s.io/queue-name' label set + awMap, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + _, err = test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Create(test.Ctx(), &unstructured.Unstructured{Object: awMap}, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Dynamic().Resource(awv1beta2.GroupVersion.WithResource("appwrappers")).Namespace(ns.Name).Delete(test.Ctx(), awName, metav1.DeleteOptions{}) + }) + }) + + /************************************************************************** + Testing the 2nd alternative behavior which targets specific namespaces that have the 'kueue-managed' label + **************************************************************************/ + t.Run("Custom ValidatingAdmissionPolicyBinding", func(t *testing.T) { + // Update the test namespace with the new 'kueue-managed' label + ns, err = test.Client().Core().CoreV1().Namespaces().Get(test.Ctx(), ns.Name, metav1.GetOptions{}) + if ns.Labels == nil { + ns.Labels = map[string]string{} + } + ns.Labels["kueue-managed"] = "true" + _, err = test.Client().Core().CoreV1().Namespaces().Update(test.Ctx(), ns, metav1.UpdateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + + // Apply the ValidatingAdmissionPolicyBinding targetting namespaces with the label 'kueue-managed' + vapb, err := test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Get(test.Ctx(), vapb.Name, metav1.GetOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + + vapb.Spec.MatchResources.NamespaceSelector.MatchLabels = map[string]string{"kueue-managed": "true"} + _, err = test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Update(test.Ctx(), vapb, metav1.UpdateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + time.Sleep(2 * time.Second) // Wait for the ValidatingAdmissionPolicyBinding to be updated + defer revertVAPB(test, vapbCopy) + + t.Run("RayCluster should be admitted with the 'kueue.x-k8s.io/queue-name' label in a labeled namespace", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcWithLQName), ns.Name).Queue(lq.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("RayCluster should not be admitted without the 'kueue.x-k8s.io/queue-name' label in a labeled namespace", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcNoLQName), ns.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters(ns.Name).Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(BeNil()) + defer test.Client().Ray().RayV1().RayClusters(ns.Name).Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("RayCluster should be admitted with the 'kueue.x-k8s.io/queue-name' label in any other namespace", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcWithLQName), "default").Queue(lq.Name).Obj() + _, err = test.Client().Ray().RayV1().RayClusters("default").Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters("default").Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + t.Run("RayCluster should be admitted without the 'kueue.x-k8s.io/queue-name' label in any other namespace", func(t *testing.T) { + rc = testingraycluster.MakeCluster(uniqueSuffix(rcNoLQName), "default").Obj() + _, err = test.Client().Ray().RayV1().RayClusters("default").Create(test.Ctx(), rc, metav1.CreateOptions{}) + test.Expect(err).ToNot(HaveOccurred()) + defer test.Client().Ray().RayV1().RayClusters("default").Delete(test.Ctx(), rc.Name, metav1.DeleteOptions{}) + }) + }) +} + +// Revert validating-admission-policy-binding to its original state +func revertVAPB(test Test, vapbCopy *vapv1.ValidatingAdmissionPolicyBinding) { + patchBytes, _ := json.Marshal(map[string]interface{}{ + "spec": map[string]interface{}{ + "policyName": vapbCopy.Spec.PolicyName, + "matchResources": map[string]interface{}{ + "namespaceSelector": map[string]interface{}{ + "matchLabels": vapbCopy.Spec.MatchResources.NamespaceSelector.MatchLabels, + }, + }, + }, + }) + _, err := test.Client().Core().AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Patch(test.Ctx(), vapbCopy.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) + test.Expect(err).ToNot(HaveOccurred()) +}