Partial extension API server store + control over printed columns (#432)

* Checkpoint * Add support for custom columns * Remove old Store and Delegate abstraction * Fix nits and rewording * Remove unused mock file * Update documentation for extension api server * Remove the need for scheme for ConvertListOptions * Rename store to utils * fixup! Remove the need for scheme for ConvertListOptions * Move watch helper to tests * Add convertError at a few places * Ignore misspell on creater * Fix comments and remove unused params * Add convertError to missing error returns * Fix watcher implementation * Document request.UserFrom and request.NamespaceFrom
rancher · Jan 15, 2025 · fdf2ef8 · fdf2ef8
1 parent 4477e2c
commit fdf2ef8
Show file tree

Hide file tree

Showing 13 changed files with 984 additions and 2,068 deletions.
diff --git a/pkg/ext/apiserver.go b/pkg/ext/apiserver.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"sync"
 
-	"k8s.io/apimachinery/pkg/api/meta"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -72,19 +71,19 @@ type ExtensionAPIServerOptions struct {
 //
 // Use [NewExtensionAPIServer] to create an ExtensionAPIServer.
 //
-// Use [InstallStore] to add a new resource store onto an existing ExtensionAPIServer.
+// Use [ExtensionAPIServer.Install] to add a new resource store onto an existing ExtensionAPIServer.
 // Each resources will then be reachable via /apis/<group>/<version>/<resource> as
 // defined by the Kubernetes API.
 //
-// When Run() is called, a separate HTTPS server is started. This server is meant
+// When [ExtensionAPIServer.Run] is called, a separate HTTPS server is started. This server is meant
 // for the main kube-apiserver to communicate with our extension API server. We
 // can expect the following requests from the main kube-apiserver:
 //
-// <path>                 <user>                 <groups>
-// /openapi/v2            system:aggregator      [system:authenticated]
-// /openapi/v3            system:aggregator      [system:authenticated]
-// /apis                  system:kube-aggregator [system:masters system:authenticated]
-// /apis/ext.cattle.io/v1 system:kube-aggregator [system:masters system:authenticated]
+//	<path>                 <user>                 <groups>
+//	/openapi/v2            system:aggregator      [system:authenticated]
+//	/openapi/v3            system:aggregator      [system:authenticated]
+//	/apis                  system:kube-aggregator [system:masters system:authenticated]
+//	/apis/ext.cattle.io/v1 system:kube-aggregator [system:masters system:authenticated]
 type ExtensionAPIServer struct {
 	codecs serializer.CodecFactory
 	scheme *runtime.Scheme
@@ -201,34 +200,49 @@ func (s *ExtensionAPIServer) ServeHTTP(w http.ResponseWriter, req *http.Request)
 	s.handler.ServeHTTP(w, req)
 }
 
-// InstallStore installs a store on the given ExtensionAPIServer object.
+// GetAuthorizer returns the authorizer used by the extension server to authorize
+// requests
 //
-// t and TList must be non-nil.
+// This can be used to inject the authorizer in stores that need them.
+func (s *ExtensionAPIServer) GetAuthorizer() authorizer.Authorizer {
+	return s.authorizer
+}
+
+// Install adds a new store to the extension API server.
 //
-// Here's an example store for a Token and TokenList resource in the ext.cattle.io/v1 apiVersion:
+// A store implements handlers for the various operations (verbs) supported for
+// a defined GVK / GVR. For example, a store for a (apiVersion:
+// ext.cattle.io/v1, kind: Tokens) Custom Resource could implement create and
+// watch verbs.
 //
-//	gvk := schema.GroupVersionKind{
-//		Group: "ext.cattle.io",
-//		Version: "v1",
-//		Kind: "Token",
-//	}
-//	InstallStore(s, &Token{}, &TokenList{}, "tokens", "token", gvk, store)
+// A store MUST implement the following interfaces: [rest.Storage], [rest.Scoper], [rest.GroupVersionKindProvider]
+// and [rest.SingularNameProvider].
 //
-// Note: Not using a method on ExtensionAPIServer object due to Go generic limitations.
-func InstallStore[T runtime.Object, TList runtime.Object](
-	s *ExtensionAPIServer,
-	t T,
-	tList TList,
-	resourceName string,
-	singularName string,
-	gvk schema.GroupVersionKind,
-	store Store[T, TList],
-) error {
-
-	if !meta.IsListType(tList) {
-		return fmt.Errorf("tList (%T) is not a list type", tList)
-	}
-
+// Implementing the various verbs goes as follows:
+//   - get: [rest.Getter] must be implemented
+//   - list: [rest.Lister] must be implemented. To help implement table conversion, we provide [ConvertToTable] and [ConvertToTableDefault].
+//     Use [ConvertListOptions] to convert the [metainternalversion.ListOptions] to a [metav1.ListOptions].
+//   - watch: [rest.Watcher] must be implemented. Use [ConvertListOptions] to convert the [metainternalversion.ListOptions] to a [metav1.ListOptions].
+//   - create: [rest.Creater] must be implemented
+//   - update: [rest.Updater] must be implemented. To help implement this correctly with create-on-update support, we provide [CreateOrUpdate].
+//   - patch: [rest.Patcher] must be implemented, which is essentially [rest.Getter] and [rest.Updater]
+//   - delete: [rest.GracefulDeleter] must be implemented
+//   - deletecollection: [rest.CollectionDeleter] must be implemented
+//
+// Most of these methods have a [context.Context] parameter that can be used to get more information
+// about the request. Here are some examples:
+//   - [request.UserFrom] to get the user info
+//   - [request.NamespaceFrom] to get the namespace (if applicable)
+//
+// For an example store implementing these, please look at the testStore type with the caveat that it is a dummy test-special purpose
+// store.
+//
+// Note that errors returned by any operations above MUST be of type [k8s.io/apimachinery/pkg/api/errors.APIStatus].
+// These can be created with [k8s.io/apimachinery/pkg/api/errors.NewNotFound], etc.
+// If an error of unknown type is returned, the library will log an error message.
+//
+//nolint:misspell
+func (s *ExtensionAPIServer) Install(resourceName string, gvk schema.GroupVersionKind, storage rest.Storage) error {
 	apiGroup, ok := s.apiGroups[gvk.Group]
 	if !ok {
 		apiGroup = genericapiserver.NewDefaultAPIGroupInfo(gvk.Group, s.scheme, metav1.ParameterCodec, s.codecs)
@@ -239,25 +253,7 @@ func InstallStore[T runtime.Object, TList runtime.Object](
 		apiGroup.VersionedResourcesStorageMap[gvk.Version] = make(map[string]rest.Storage)
 	}
 
-	del := &delegateError[T, TList]{
-		inner: &delegate[T, TList]{
-			scheme: s.scheme,
-
-			t:            t,
-			tList:        tList,
-			singularName: singularName,
-			gvk:          gvk,
-			gvr: schema.GroupVersionResource{
-				Group:    gvk.Group,
-				Version:  gvk.Version,
-				Resource: resourceName,
-			},
-			authorizer: s.authorizer,
-			store:      store,
-		},
-	}
-
-	apiGroup.VersionedResourcesStorageMap[gvk.Version][resourceName] = del
+	apiGroup.VersionedResourcesStorageMap[gvk.Version][resourceName] = storage
 	s.apiGroups[gvk.Group] = apiGroup
 	return nil
 }

diff --git a/pkg/ext/apiserver_authentication_test.go b/pkg/ext/apiserver_authentication_test.go
@@ -1,6 +1,7 @@
 package ext
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,22 +13,32 @@ import (
 
 	"github.com/stretchr/testify/require"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/server/options"
 )
 
+var _ rest.Storage = (*authnTestStore)(nil)
+var _ rest.Lister = (*authnTestStore)(nil)
+
 type authnTestStore struct {
-	*testStore
+	*testStore[*TestType, *TestTypeList]
 	userCh chan user.Info
 }
 
-func (t *authnTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeList, error) {
-	t.userCh <- ctx.User
+func (t *authnTestStore) List(ctx context.Context, _ *metainternalversion.ListOptions) (runtime.Object, error) {
+	userInfo, ok := request.UserFrom(ctx)
+	if !ok {
+		return nil, convertError(fmt.Errorf("missing user info"))
+	}
+
+	t.userCh <- userInfo
 	return &testTypeListFixture, nil
 }
 
@@ -50,10 +61,10 @@ func TestAuthenticationCustom(t *testing.T) {
 	require.NoError(t, err)
 
 	store := &authnTestStore{
-		testStore: &testStore{},
+		testStore: newDefaultTestStore(),
 		userCh:    make(chan user.Info, 100),
 	}
-	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
 		opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {

diff --git a/pkg/ext/apiserver_authorization_test.go b/pkg/ext/apiserver_authorization_test.go
@@ -23,32 +23,40 @@ import (
 	"go.uber.org/mock/gomock"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/server/options"
 )
 
 type authzTestStore struct {
-	*testStore
+	*testStore[*TestType, *TestTypeList]
+	authorizer authorizer.Authorizer
 }
 
-func (t *authzTestStore) Get(ctx Context, name string, opts *metav1.GetOptions) (*TestType, error) {
-	if name == "not-found" {
-		return nil, apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
-	}
-	return t.testStore.Get(ctx, name, opts)
+// Get implements [rest.Getter]
+func (t *authzTestStore) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
+	return t.get(ctx, name, options)
 }
 
-func (t *authzTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeList, error) {
-	if ctx.User.GetName() == "read-only-error" {
-		decision, _, err := ctx.Authorizer.Authorize(ctx, authorizer.AttributesRecord{
-			User:            ctx.User,
+// List implements [rest.Lister]
+func (t *authzTestStore) List(ctx context.Context, _ *metainternalversion.ListOptions) (runtime.Object, error) {
+	userInfo, ok := request.UserFrom(ctx)
+	if !ok {
+		return nil, convertError(fmt.Errorf("missing user info"))
+	}
+
+	if userInfo.GetName() == "read-only-error" {
+		decision, _, err := t.authorizer.Authorize(ctx, authorizer.AttributesRecord{
+			User:            userInfo,
 			Verb:            "customverb",
 			Resource:        "testtypes",
 			ResourceRequest: true,
@@ -58,7 +66,7 @@ func (t *authzTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeL
 			if err == nil {
 				err = fmt.Errorf("not allowed")
 			}
-			forbidden := apierrors.NewForbidden(ctx.GroupVersionResource.GroupResource(), "Forbidden", err)
+			forbidden := apierrors.NewForbidden(t.gvr.GroupResource(), "Forbidden", err)
 			forbidden.ErrStatus.Kind = "Status"
 			forbidden.ErrStatus.APIVersion = "v1"
 			return nil, forbidden
@@ -67,6 +75,54 @@ func (t *authzTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeL
 	return &testTypeListFixture, nil
 }
 
+func (t *authzTestStore) get(_ context.Context, name string, _ *metav1.GetOptions) (*TestType, error) {
+	if name == "not-found" {
+		return nil, apierrors.NewNotFound(t.gvr.GroupResource(), name)
+	}
+	return &testTypeFixture, nil
+}
+
+func (t *authzTestStore) create(_ context.Context, _ *TestType, _ *metav1.CreateOptions) (*TestType, error) {
+	return &testTypeFixture, nil
+}
+
+func (t *authzTestStore) update(_ context.Context, _ *TestType, _ *metav1.UpdateOptions) (*TestType, error) {
+	return &testTypeFixture, nil
+}
+
+// Create implements [rest.Creater]
+func (t *authzTestStore) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
+	if createValidation != nil {
+		err := createValidation(ctx, obj)
+		if err != nil {
+			return obj, err
+		}
+	}
+
+	objT, ok := obj.(*TestType)
+	if !ok {
+		var zeroT *TestType
+		return nil, convertError(fmt.Errorf("expected %T but got %T", zeroT, obj))
+	}
+
+	return t.create(ctx, objT, options)
+}
+
+// Update implements [rest.Updater]
+func (t *authzTestStore) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	return CreateOrUpdate(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options, t.get, t.create, t.update)
+}
+
+// Watch implements [rest.Watcher]
+func (t *authzTestStore) Watch(_ context.Context, _ *metainternalversion.ListOptions) (watch.Interface, error) {
+	return nil, nil
+}
+
+// Delete implements [rest.GracefulDeleter]
+func (t *authzTestStore) Delete(_ context.Context, _ string, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions) (runtime.Object, bool, error) {
+	return nil, false, nil
+}
+
 func (s *ExtensionAPIServerSuite) TestAuthorization() {
 	t := s.T()
 
@@ -89,10 +145,7 @@ func (s *ExtensionAPIServerSuite) TestAuthorization() {
 	ln, _, err := options.CreateListener("", ":0", net.ListenConfig{})
 	require.NoError(t, err)
 
-	store := &authzTestStore{
-		testStore: &testStore{},
-	}
-	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, cleanup, err := setupExtensionAPIServerNoStore(t, scheme, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authz
 		opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
@@ -104,7 +157,17 @@ func (s *ExtensionAPIServerSuite) TestAuthorization() {
 				User: user,
 			}, true, nil
 		})
-	}, nil)
+	}, func(s *ExtensionAPIServer) error {
+		store := &authzTestStore{
+			testStore:  newDefaultTestStore(),
+			authorizer: s.GetAuthorizer(),
+		}
+		err := s.Install("testtypes", testTypeGV.WithKind("TestType"), store)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
 	require.NoError(t, err)
 	defer cleanup()