Readd k3s secrets-encrypt rotate-keys with correct support for KMSv2 GA

.drone.yml

@@ -651,6 +651,10 @@ steps:
   - vagrant destroy -f
   - go test -v -timeout=45m ./validatecluster_test.go -ci -local
   - cp ./coverage.out /tmp/artifacts/validate-coverage.out
+  - cd ../secretsencryption
+  - vagrant destroy -f
+  - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+  - cp ./coverage.out /tmp/artifacts/se-coverage.out
   - cd ../startup
   - vagrant destroy -f
   - go test -v -timeout=30m ./startup_test.go -ci -local

Dockerfile.local

@@ -46,9 +46,9 @@ RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
     ./scripts/download
 
 COPY ./cmd ./cmd
-COPY ./pkg ./pkg
 COPY ./tests ./tests
 COPY ./.git ./.git
+COPY ./pkg ./pkg
 RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
     --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
     ./scripts/build

go.mod

@@ -136,6 +136,7 @@ require (
 	github.com/opencontainers/selinux v1.11.0
 	github.com/otiai10/copy v1.7.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/common v0.45.0
 	github.com/rancher/dynamiclistener v0.3.6
 	github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29
 	github.com/rancher/remotedialer v0.3.0
@@ -420,7 +421,6 @@ require (
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect

pkg/cli/cmds/secrets_encrypt.go

@@ -84,6 +84,13 @@ func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencry
 						Destination: &ServerConfig.EncryptSkip,
 					}),
 			},
+			{
+				Name:           "rotate-keys",
+				Usage:          "(experimental) Dynamically rotates secrets encryption keys and re-encrypt secrets",
+				SkipArgReorder: true,
+				Action:         rotateKeys,
+				Flags:          EncryptFlags,
+			},
 		},
 	}
 }

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 	"text/tabwriter"
+	"time"
 
 	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -226,7 +227,11 @@ func RotateKeys(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+	timeout := 70 * time.Second
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config",
+		b,
+		clientaccess.WithTimeout(timeout),
+		clientaccess.WithHeaderTimeout(timeout)); err != nil {
 		return wrapServerError(err)
 	}
 	fmt.Println("keys rotated, reencryption started")

pkg/clientaccess/token.go

@@ -41,6 +41,9 @@ var (
 	}
 )
 
+// ClientOption is a callback to mutate the http client prior to use
+type ClientOption func(*http.Client)
+
 // Info contains fields that track parsed parts of a cluster join token
 type Info struct {
 	*kubeadm.BootstrapTokenString
@@ -233,7 +236,7 @@ func parseToken(token string) (*Info, error) {
 // If the CA bundle is not empty but does not contain any valid certs, it validates using
 // an empty CA bundle (which will always fail).
 // If valid cert+key paths can be loaded from the provided paths, they are used for client cert auth.
-func GetHTTPClient(cacerts []byte, certFile, keyFile string) *http.Client {
+func GetHTTPClient(cacerts []byte, certFile, keyFile string, option ...ClientOption) *http.Client {
 	if len(cacerts) == 0 {
 		return defaultClient
 	}
@@ -250,18 +253,34 @@ func GetHTTPClient(cacerts []byte, certFile, keyFile string) *http.Client {
 	if err == nil {
 		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
-
-	return &http.Client{
+	client := &http.Client{
 		Timeout: defaultClientTimeout,
 		Transport: &http.Transport{
 			DisableKeepAlives: true,
 			TLSClientConfig:   tlsConfig,
 		},
 	}
+
+	for _, o := range option {
+		o(client)
+	}
+	return client
+}
+
+func WithTimeout(d time.Duration) ClientOption {
+	return func(c *http.Client) {
+		c.Timeout = d
+	}
+}
+
+func WithHeaderTimeout(d time.Duration) ClientOption {
+	return func(c *http.Client) {
+		c.Transport.(*http.Transport).ResponseHeaderTimeout = d
+	}
 }
 
 // Get makes a request to a subpath of info's BaseURL
-func (i *Info) Get(path string) ([]byte, error) {
+func (i *Info) Get(path string, option ...ClientOption) ([]byte, error) {
 	u, err := url.Parse(i.BaseURL)
 	if err != nil {
 		return nil, err
@@ -272,11 +291,12 @@ func (i *Info) Get(path string) ([]byte, error) {
 	}
 	p.Scheme = u.Scheme
 	p.Host = u.Host
-	return get(p.String(), GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile), i.Username, i.Password, i.Token())
+	return get(p.String(), GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile, option...), i.Username, i.Password, i.Token())
 }
 
-// Put makes a request to a subpath of info's BaseURL
-func (i *Info) Put(path string, body []byte) error {
+// Put makes a request to a subpath of info's BaseURL, with a 10 second timeout
+func (i *Info) Put(path string, body []byte, option ...ClientOption) error {
+
 	u, err := url.Parse(i.BaseURL)
 	if err != nil {
 		return err
@@ -287,7 +307,10 @@ func (i *Info) Put(path string, body []byte) error {
 	}
 	p.Scheme = u.Scheme
 	p.Host = u.Host
-	return put(p.String(), body, GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile), i.Username, i.Password, i.Token())
+
+	client := GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile, option...)
+
+	return put(p.String(), body, client, i.Username, i.Password, i.Token())
 }
 
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server

pkg/daemons/control/deps/deps.go

@@ -741,7 +741,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
 		return nil
 	}
 
-	aescbcKey := make([]byte, aescbcKeySize, aescbcKeySize)
+	aescbcKey := make([]byte, aescbcKeySize)
 	_, err := cryptorand.Read(aescbcKey)
 	if err != nil {
 		return err

pkg/daemons/control/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -60,6 +61,19 @@ func Server(ctx context.Context, cfg *config.Control) error {
 		if err := apiServer(ctx, cfg); err != nil {
 			return err
 		}
+		if cfg.EncryptSecrets {
+			controllerName := "reencrypt-secrets"
+			cfg.Runtime.ClusterControllerStarts[controllerName] = func(ctx context.Context) {
+				// cfg.Runtime.Core is populated before this callback is triggered
+				if err := secretsencrypt.Register(ctx,
+					controllerName,
+					cfg,
+					cfg.Runtime.Core.Core().V1().Node(),
+					cfg.Runtime.Core.Core().V1().Secret()); err != nil {
+					logrus.Errorf("Failed to register %s controller: %v", controllerName, err)
+				}
+			}
+		}
 	}
 
 	// Wait for an apiserver to become available before starting additional controllers,

pkg/secretsencrypt/config.go

@@ -1,20 +1,29 @@
 package secretsencrypt
 
 import (
+	"bytes"
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/prometheus/common/expfmt"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/clientcmd"
 
+	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+
+	"k8s.io/client-go/rest"
 )
 
 const (
@@ -42,7 +51,10 @@ func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1
 	return curEncryption.Resources[0].Providers, nil
 }
 
-func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) {
+// GetEncryptionKeys returns a list of encryption keys from the current encryption configuration.
+// If includeIdentity is true, it will also include a fake key representing the identity provider, which
+// is used to determine if encryption is enabled/disabled.
+func GetEncryptionKeys(runtime *config.ControlRuntime, includeIdentity bool) ([]apiserverconfigv1.Key, error) {
 
 	providers, err := GetEncryptionProviders(runtime)
 	if err != nil {
@@ -54,6 +66,14 @@ func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key,
 
 	var curKeys []apiserverconfigv1.Key
 	for _, p := range providers {
+		// Since identity doesn't have keys, we make up a fake key to represent it, so we can
+		// know that encryption is enabled/disabled in the request.
+		if p.Identity != nil && includeIdentity {
+			curKeys = append(curKeys, apiserverconfigv1.Key{
+				Name:   "identity",
+				Secret: "identity",
+			})
+		}
 		if p.AESCBC != nil {
 			curKeys = append(curKeys, p.AESCBC.Keys...)
 		}
@@ -121,10 +141,10 @@ func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
 }
 
 // GenReencryptHash generates a sha256 hash from the existing secrets keys and
-// a new key based on the input arguments.
+// any identity providers plus a new key based on the input arguments.
 func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
 
-	keys, err := GetEncryptionKeys(runtime)
+	keys, err := GetEncryptionKeys(runtime, true)
 	if err != nil {
 		return "", err
 	}
@@ -174,3 +194,93 @@ func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.
 	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
 	return os.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
 }
+
+// WaitForEncryptionConfigReload watches the metrics API, polling the latest time the encryption config was reloaded.
+func WaitForEncryptionConfigReload(runtime *config.ControlRuntime, reloadSuccesses, reloadTime int64) error {
+	var lastFailure string
+	err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
+
+		newReloadTime, newReloadSuccess, err := GetEncryptionConfigMetrics(runtime, false)
+		if err != nil {
+			return true, err
+		}
+
+		if newReloadSuccess <= reloadSuccesses || newReloadTime <= reloadTime {
+			lastFailure = fmt.Sprintf("apiserver has not reloaded encryption configuration (reload success: %d/%d, reload timestamp %d/%d)", newReloadSuccess, reloadSuccesses, newReloadTime, reloadTime)
+			return false, nil
+		}
+		logrus.Infof("encryption config reloaded successfully %d times", newReloadSuccess)
+		logrus.Debugf("encryption config reloaded at %s", time.Unix(newReloadTime, 0))
+		return true, nil
+	})
+	if err != nil {
+		err = fmt.Errorf("%w: %s", err, lastFailure)
+	}
+	return err
+}
+
+// GetEncryptionConfigMetrics fetches the metrics API and returns the last time the encryption config was reloaded
+// and the number of times it has been reloaded.
+func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics bool) (int64, int64, error) {
+	var unixUpdateTime int64
+	var reloadSuccessCounter int64
+	var lastFailure string
+	restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigSupervisor)
+	if err != nil {
+		return 0, 0, err
+	}
+	restConfig.GroupVersion = &apiserverconfigv1.SchemeGroupVersion
+	restConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
+	restClient, err := rest.RESTClientFor(restConfig)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// This is wrapped in a poller because on startup no metrics exist. Its only after the encryption config
+	// is modified and the first reload occurs that the metrics are available.
+	err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
+		data, err := restClient.Get().AbsPath("/metrics").DoRaw(context.TODO())
+		if err != nil {
+			return true, err
+		}
+
+		reader := bytes.NewReader(data)
+		var parser expfmt.TextParser
+		mf, err := parser.TextToMetricFamilies(reader)
+		if err != nil {
+			return true, err
+		}
+		tsMetric := mf["apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds"]
+		successMetric := mf["apiserver_encryption_config_controller_automatic_reload_success_total"]
+
+		// First time, no metrics exist, so return zeros
+		if tsMetric == nil && successMetric == nil && initialMetrics {
+			return true, nil
+		}
+
+		if tsMetric == nil {
+			lastFailure = "encryption config time metric not found"
+			return false, nil
+		}
+
+		if successMetric == nil {
+			lastFailure = "encryption config success metric not found"
+			return false, nil
+		}
+
+		unixUpdateTime = int64(tsMetric.GetMetric()[0].GetGauge().GetValue())
+		if time.Now().Unix() < unixUpdateTime {
+			return true, fmt.Errorf("encryption reload time is incorrectly ahead of current time")
+		}
+
+		reloadSuccessCounter = int64(successMetric.GetMetric()[0].GetCounter().GetValue())
+
+		return true, nil
+	})
+
+	if err != nil {
+		err = fmt.Errorf("%w: %s", err, lastFailure)
+	}
+
+	return unixUpdateTime, reloadSuccessCounter, err
+}