From 8f661fddbb04b3aa350c195448dae486183e49ab Mon Sep 17 00:00:00 2001 From: Michi Mutsuzaki Date: Thu, 5 Oct 2023 22:20:08 +0000 Subject: [PATCH] Remove the "operator" init container - Remove the "operator" init container. Now there is a proper operator deployment, so we don't need to create CRDs in the init container. - Update the Tetragon daemonset ClusterRole accordingly. - Modify the Tetragon daemonset initialization logic to wait for all the required CRDs to show up before proceeding. Signed-off-by: Michi Mutsuzaki --- cmd/tetragon/flags.go | 4 + cmd/tetragon/main.go | 84 ++++++++++++----- .../templates/_container_tetragon.tpl | 11 --- install/kubernetes/templates/clusterrole.yaml | 18 +--- install/kubernetes/templates/daemonset.yaml | 7 -- .../templates/tetragon_configmap.yaml | 1 + pkg/option/config.go | 2 + .../v1/customresourcedefinition.go | 89 +++++++++++++++++++ .../apiextensions/v1/interface.go | 45 ++++++++++ .../internalinterfaces/factory_interfaces.go | 40 +++++++++ .../v1/customresourcedefinition.go | 68 ++++++++++++++ .../apiextensions/v1/expansion_generated.go | 23 +++++ vendor/modules.txt | 3 + 13 files changed, 338 insertions(+), 57 deletions(-) create mode 100644 vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/customresourcedefinition.go create mode 100644 vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/interface.go create mode 100644 vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go create mode 100644 vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/customresourcedefinition.go create mode 100644 vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/expansion_generated.go diff --git a/cmd/tetragon/flags.go b/cmd/tetragon/flags.go index b6ef0c07fe4..f65b599db98 100644 --- a/cmd/tetragon/flags.go +++ b/cmd/tetragon/flags.go @@ -80,6 +80,8 @@ const ( keyEnableMsgHandlingLatency = "enable-msg-handling-latency" keyKmods = "kmods" + + keyEnablePodInfo = "enable-pod-info" ) func readAndSetFlags() { @@ -144,6 +146,8 @@ func readAndSetFlags() { option.Config.KMods = viper.GetStringSlice(keyKmods) + option.Config.EnablePodInfo = viper.GetBool(keyEnablePodInfo) + if viper.IsSet(keyTracingPolicy) { option.Config.TracingPolicy = viper.GetString(keyTracingPolicy) } diff --git a/cmd/tetragon/main.go b/cmd/tetragon/main.go index 45e4609dbf5..da1c582a7d8 100644 --- a/cmd/tetragon/main.go +++ b/cmd/tetragon/main.go @@ -53,13 +53,18 @@ import ( _ "github.com/cilium/tetragon/pkg/sensors" "github.com/cilium/lumberjack/v2" + "github.com/cilium/tetragon/pkg/k8s/apis/cilium.io/v1alpha1" gops "github.com/google/gops/agent" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/durationpb" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsinformer "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" ) var ( @@ -310,16 +315,64 @@ func tetragonExecute() error { // Probe runtime configuration and do not fail on errors obs.UpdateRuntimeConf(option.Config.MapDir) - watcher, err := getWatcher() - if err != nil { - return err - } - _, err = cilium.InitCiliumState(ctx, option.Config.EnableCilium) + var k8sWatcher watcher.K8sResourceWatcher + if option.Config.EnableK8s { + log.Info("Enabling Kubernetes API") + crds := map[string]struct{}{ + v1alpha1.TPName: {}, + v1alpha1.TPNamespacedName: {}, + } + if option.Config.EnablePodInfo { + crds[v1alpha1.PIName] = struct{}{} + } + config, err := k8sconf.K8sConfig() + if err != nil { + return err + } + log.WithField("crds", crds).Info("Waiting for required CRDs") + var wg sync.WaitGroup + wg.Add(1) + k8sClient := kubernetes.NewForConfigOrDie(config) + crdClient := apiextensionsclientset.NewForConfigOrDie(config) + crdInformer := apiextensionsinformer.NewCustomResourceDefinitionInformer(crdClient, 0*time.Second, nil) + _, err = crdInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + crdObject, ok := obj.(*v1.CustomResourceDefinition) + if !ok { + log.WithField("obj", obj).Warn("Received an invalid object") + return + } + if _, ok := crds[crdObject.Name]; ok { + log.WithField("crd", crdObject.Name).Info("Found CRD") + delete(crds, crdObject.Name) + if len(crds) == 0 { + log.Info("Found all the required CRDs") + wg.Done() + } + } + }, + }) + if err != nil { + log.WithError(err).Error("failed to add event handler") + return err + } + stop := make(chan struct{}) + go func() { + crdInformer.Run(stop) + }() + wg.Wait() + close(stop) + k8sWatcher = watcher.NewK8sWatcher(k8sClient, 60*time.Second) + } else { + log.Info("Disabling Kubernetes API") + k8sWatcher = watcher.NewFakeK8sWatcher(nil) + } + _, err := cilium.InitCiliumState(ctx, option.Config.EnableCilium) if err != nil { return err } - if err := process.InitCache(watcher, option.Config.ProcessCacheSize); err != nil { + if err := process.InitCache(k8sWatcher, option.Config.ProcessCacheSize); err != nil { return err } @@ -338,7 +391,7 @@ func tetragonExecute() error { ctx, cancel2 := context.WithCancel(ctx) defer cancel2() - hookRunner := rthooks.GlobalRunner().WithWatcher(watcher) + hookRunner := rthooks.GlobalRunner().WithWatcher(k8sWatcher) pm, err := tetragonGrpc.NewProcessManager( ctx, @@ -631,21 +684,6 @@ func Serve(ctx context.Context, listenAddr string, srv *server.Server) error { return nil } -func getWatcher() (watcher.K8sResourceWatcher, error) { - if option.Config.EnableK8s { - log.Info("Enabling Kubernetes API") - config, err := k8sconf.K8sConfig() - if err != nil { - return nil, err - } - k8sClient := kubernetes.NewForConfigOrDie(config) - return watcher.NewK8sWatcher(k8sClient, 60*time.Second), nil - - } - log.Info("Disabling Kubernetes API") - return watcher.NewFakeK8sWatcher(nil), nil -} - func startGopsServer() error { // Empty means no gops if option.Config.GopsAddr == "" { @@ -773,6 +811,8 @@ func execute() error { flags.Int(keyRBQueueSize, 65535, "Set size of channel between ring buffer and sensor go routines (default 65k)") + flags.Bool(keyEnablePodInfo, false, "Enable PodInfo custom resource") + viper.BindPFlags(flags) return rootCmd.Execute() } diff --git a/install/kubernetes/templates/_container_tetragon.tpl b/install/kubernetes/templates/_container_tetragon.tpl index e4b9377619b..04bdfb6060e 100644 --- a/install/kubernetes/templates/_container_tetragon.tpl +++ b/install/kubernetes/templates/_container_tetragon.tpl @@ -80,14 +80,3 @@ {{- end -}} {{- end -}} -{{- define "container.tetragon.init-operator" -}} -- name: {{ include "container.tetragon.name" . }}-operator - image: "{{ if .Values.tetragonOperator.image.override }}{{ .Values.tetragonOperator.image.override }}{{ else }}{{ .Values.tetragonOperator.image.repository }}{{ .Values.tetragonOperator.image.suffix }}:{{ .Values.tetragonOperator.image.tag }}{{ end }}" - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - --config-dir=/etc/tetragon/operator.conf.d/ - volumeMounts: - - mountPath: /etc/tetragon/operator.conf.d/ - name: tetragon-operator-config - readOnly: true -{{- end -}} diff --git a/install/kubernetes/templates/clusterrole.yaml b/install/kubernetes/templates/clusterrole.yaml index e687a388aed..3750aaf97f1 100644 --- a/install/kubernetes/templates/clusterrole.yaml +++ b/install/kubernetes/templates/clusterrole.yaml @@ -31,23 +31,7 @@ rules: resources: - customresourcedefinitions verbs: - - create - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - resourceNames: - - tracingpolicies.cilium.io - - tracingpoliciesnamespaced.cilium.io - - podinfo.cilium.io - verbs: - - update - get - list + - watch {{- end }} diff --git a/install/kubernetes/templates/daemonset.yaml b/install/kubernetes/templates/daemonset.yaml index 23d7ecbda0b..01ecb9abc51 100644 --- a/install/kubernetes/templates/daemonset.yaml +++ b/install/kubernetes/templates/daemonset.yaml @@ -44,10 +44,6 @@ spec: securityContext: {{- toYaml . | nindent 6 }} {{- end }} -{{- if .Values.tetragon.enabled }} - initContainers: - {{- include "container.tetragon.init-operator" . | nindent 6 -}} -{{- end }} containers: {{- if eq .Values.export.mode "stdout" }} {{- include "container.export.stdout" . | nindent 6 -}} @@ -96,9 +92,6 @@ spec: name: metadata-files {{- end }} {{- end }} - - name: tetragon-operator-config - configMap: - name: {{ .Release.Name }}-operator-config {{- with .Values.extraVolumes }} {{- toYaml . | nindent 6 }} {{- end }} diff --git a/install/kubernetes/templates/tetragon_configmap.yaml b/install/kubernetes/templates/tetragon_configmap.yaml index 3fd219ef60c..1da1ab40310 100644 --- a/install/kubernetes/templates/tetragon_configmap.yaml +++ b/install/kubernetes/templates/tetragon_configmap.yaml @@ -54,3 +54,4 @@ data: {{- if .Values.tetragon.enableMsgHandlingLatency }} enable-msg-handling-latency: "true" {{- end }} + enable-pod-info: {{ .Values.tetragonOperator.podInfo.enabled | quote }} diff --git a/pkg/option/config.go b/pkg/option/config.go index 06dadb6723c..f0c2caac0a7 100644 --- a/pkg/option/config.go +++ b/pkg/option/config.go @@ -81,6 +81,8 @@ type config struct { EnableMsgHandlingLatency bool KMods []string + + EnablePodInfo bool } var ( diff --git a/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/customresourcedefinition.go b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/customresourcedefinition.go new file mode 100644 index 00000000000..7d1b5711126 --- /dev/null +++ b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/customresourcedefinition.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + internalinterfaces "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces" + v1 "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// CustomResourceDefinitionInformer provides access to a shared informer and lister for +// CustomResourceDefinitions. +type CustomResourceDefinitionInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.CustomResourceDefinitionLister +} + +type customResourceDefinitionInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewCustomResourceDefinitionInformer constructs a new informer for CustomResourceDefinition type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewCustomResourceDefinitionInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredCustomResourceDefinitionInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredCustomResourceDefinitionInformer constructs a new informer for CustomResourceDefinition type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredCustomResourceDefinitionInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiextensionsV1().CustomResourceDefinitions().List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiextensionsV1().CustomResourceDefinitions().Watch(context.TODO(), options) + }, + }, + &apiextensionsv1.CustomResourceDefinition{}, + resyncPeriod, + indexers, + ) +} + +func (f *customResourceDefinitionInformer) defaultInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredCustomResourceDefinitionInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *customResourceDefinitionInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiextensionsv1.CustomResourceDefinition{}, f.defaultInformer) +} + +func (f *customResourceDefinitionInformer) Lister() v1.CustomResourceDefinitionLister { + return v1.NewCustomResourceDefinitionLister(f.Informer().GetIndexer()) +} diff --git a/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/interface.go b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/interface.go new file mode 100644 index 00000000000..d96e2099aec --- /dev/null +++ b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1/interface.go @@ -0,0 +1,45 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + internalinterfaces "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // CustomResourceDefinitions returns a CustomResourceDefinitionInformer. + CustomResourceDefinitions() CustomResourceDefinitionInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// CustomResourceDefinitions returns a CustomResourceDefinitionInformer. +func (v *version) CustomResourceDefinitions() CustomResourceDefinitionInformer { + return &customResourceDefinitionInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 00000000000..da6eadaa7f9 --- /dev/null +++ b/vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,40 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes clientset.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(clientset.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/customresourcedefinition.go b/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/customresourcedefinition.go new file mode 100644 index 00000000000..d83c58bc629 --- /dev/null +++ b/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/customresourcedefinition.go @@ -0,0 +1,68 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// CustomResourceDefinitionLister helps list CustomResourceDefinitions. +// All objects returned here must be treated as read-only. +type CustomResourceDefinitionLister interface { + // List lists all CustomResourceDefinitions in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.CustomResourceDefinition, err error) + // Get retrieves the CustomResourceDefinition from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.CustomResourceDefinition, error) + CustomResourceDefinitionListerExpansion +} + +// customResourceDefinitionLister implements the CustomResourceDefinitionLister interface. +type customResourceDefinitionLister struct { + indexer cache.Indexer +} + +// NewCustomResourceDefinitionLister returns a new CustomResourceDefinitionLister. +func NewCustomResourceDefinitionLister(indexer cache.Indexer) CustomResourceDefinitionLister { + return &customResourceDefinitionLister{indexer: indexer} +} + +// List lists all CustomResourceDefinitions in the indexer. +func (s *customResourceDefinitionLister) List(selector labels.Selector) (ret []*v1.CustomResourceDefinition, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.CustomResourceDefinition)) + }) + return ret, err +} + +// Get retrieves the CustomResourceDefinition from the index for a given name. +func (s *customResourceDefinitionLister) Get(name string) (*v1.CustomResourceDefinition, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("customresourcedefinition"), name) + } + return obj.(*v1.CustomResourceDefinition), nil +} diff --git a/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/expansion_generated.go b/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/expansion_generated.go new file mode 100644 index 00000000000..609d86be39f --- /dev/null +++ b/vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1/expansion_generated.go @@ -0,0 +1,23 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +// CustomResourceDefinitionListerExpansion allows custom methods to be added to +// CustomResourceDefinitionLister. +type CustomResourceDefinitionListerExpansion interface{} diff --git a/vendor/modules.txt b/vendor/modules.txt index 257669b79b8..a719c640f5f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1103,6 +1103,9 @@ k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextension k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1/fake k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake +k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1 +k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/internalinterfaces +k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1 k8s.io/apiextensions-apiserver/pkg/features # k8s.io/apimachinery v0.28.2 ## explicit; go 1.20