feat: Support ForcePrecompiled flag

Mofed images support some OS/Kernel with precompiled drivers. In case that a Precompiled image is available in the image registry, it will be used for MOFED, otherwise the image with sources will be used. The user can specify that in case the precompiled image does not exists, the Ofed State will fail. This can be done with setting ForcePrecompiled to "true" in "ofedDriver" spec in NicClusterPolicy. The default for ForcePrecompiled is false. Signed-off-by: Fred Rolland <[email protected]>
Mellanox · Mar 13, 2024 · 3cb012d · 3cb012d
1 parent 7193b1e
commit 3cb012d
Show file tree

Hide file tree

Showing 19 changed files with 457 additions and 36 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -78,6 +78,8 @@ COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/kubectl-${ARCH} /usr/local/bin/kubectl
 COPY --from=builder /workspace/crds /crds
 
+# Default Certificates are missing in micro-ubi. These are need to fetch DOCA drivers image tags
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 COPY /webhook-schemas /webhook-schemas
 COPY manifests/ manifests/
 USER 65532:65532

diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -106,6 +106,12 @@ type OFEDDriverSpec struct {
 	// +kubebuilder:default:=300
 	// +kubebuilder:validation:Minimum:=0
 	TerminationGracePeriodSeconds int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// ForcePrecompiled specifies if only MOFED precompiled images are allowed
+	// If set to false and precompiled image does not exists, MOFED drivers will be compiled on Nodes
+	// If set to true and precompiled image does not exists, OFED state will be Error.
+	// +optional
+	// +kubebuilder:default:=false
+	ForcePrecompiled bool `json:"forcePrecompiled,omitempty"`
 }
 
 // DriverUpgradePolicySpec describes policy configuration for automatic upgrades

diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
@@ -668,6 +668,13 @@ spec:
                       - name
                       type: object
                     type: array
+                  forcePrecompiled:
+                    default: false
+                    description: |-
+                      ForcePrecompiled specifies if only MOFED precompiled images are allowed
+                      If set to false and precompiled image does not exists, MOFED drivers will be compiled on Nodes
+                      If set to true and precompiled image does not exists, OFED state will be Error.
+                    type: boolean
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
                     type: string

diff --git a/controllers/nicclusterpolicy_controller.go b/controllers/nicclusterpolicy_controller.go
@@ -41,6 +41,7 @@ import (
 	"github.com/Mellanox/network-operator/pkg/clustertype"
 	"github.com/Mellanox/network-operator/pkg/config"
 	"github.com/Mellanox/network-operator/pkg/consts"
+	"github.com/Mellanox/network-operator/pkg/docadriverimages"
 	"github.com/Mellanox/network-operator/pkg/nodeinfo"
 	"github.com/Mellanox/network-operator/pkg/state"
 	"github.com/Mellanox/network-operator/pkg/staticconfig"
@@ -49,10 +50,11 @@ import (
 // NicClusterPolicyReconciler reconciles a NicClusterPolicy object
 type NicClusterPolicyReconciler struct {
 	client.Client
-	Scheme               *runtime.Scheme
-	ClusterTypeProvider  clustertype.Provider
-	StaticConfigProvider staticconfig.Provider
-	MigrationCh          chan struct{}
+	Scheme                   *runtime.Scheme
+	ClusterTypeProvider      clustertype.Provider
+	StaticConfigProvider     staticconfig.Provider
+	MigrationCh              chan struct{}
+	DocaDriverImagesProvider docadriverimages.Provider
 
 	stateManager state.Manager
 }
@@ -129,6 +131,7 @@ func (r *NicClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	sc := state.NewInfoCatalog()
 	sc.Add(state.InfoTypeClusterType, r.ClusterTypeProvider)
 	sc.Add(state.InfoTypeStaticConfig, r.StaticConfigProvider)
+
 	if instance.Spec.OFEDDriver != nil {
 		// Create node infoProvider and add to the service catalog
 		reqLogger.V(consts.LogLevelInfo).Info("Creating Node info provider")
@@ -148,6 +151,10 @@ func (r *NicClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		reqLogger.V(consts.LogLevelDebug).Info("Node info provider with", "Nodes:", nodeNames)
 		infoProvider := nodeinfo.NewProvider(nodePtrList)
 		sc.Add(state.InfoTypeNodeInfo, infoProvider)
+		r.DocaDriverImagesProvider.SetImageSpec(&instance.Spec.OFEDDriver.ImageSpec)
+		sc.Add(state.InfoTypeDocaDriverImage, r.DocaDriverImagesProvider)
+	} else {
+		r.DocaDriverImagesProvider.SetImageSpec(nil)
 	}
 	// Sync state and update status
 	managerStatus := r.stateManager.SyncState(ctx, instance, sc)

diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -39,6 +39,7 @@ import (
 
 	mellanoxcomv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1"
 	"github.com/Mellanox/network-operator/pkg/clustertype"
+	"github.com/Mellanox/network-operator/pkg/docadriverimages"
 	"github.com/Mellanox/network-operator/pkg/staticconfig"
 	// +kubebuilder:scaffold:imports
 )
@@ -133,13 +134,15 @@ var _ = BeforeSuite(func() {
 	clusterTypeProvider, err := clustertype.NewProvider(context.Background(), k8sClient)
 	Expect(err).NotTo(HaveOccurred())
 	staticConfigProvider := staticconfig.NewProvider(staticconfig.StaticConfig{CniBinDirectory: "/opt/cni/bin"})
+	docaImagesProvider := docadriverimages.NewProvider(context.Background(), k8sClient)
 
 	err = (&NicClusterPolicyReconciler{
-		Client:               k8sManager.GetClient(),
-		Scheme:               k8sManager.GetScheme(),
-		ClusterTypeProvider:  clusterTypeProvider,
-		StaticConfigProvider: staticConfigProvider,
-		MigrationCh:          migrationCompletionChan,
+		Client:                   k8sManager.GetClient(),
+		Scheme:                   k8sManager.GetScheme(),
+		ClusterTypeProvider:      clusterTypeProvider,
+		StaticConfigProvider:     staticConfigProvider,
+		MigrationCh:              migrationCompletionChan,
+		DocaDriverImagesProvider: docaImagesProvider,
 	}).SetupWithManager(k8sManager, testSetupLog)
 	Expect(err).ToNot(HaveOccurred())
 

diff --git a/deployment/network-operator/README.md b/deployment/network-operator/README.md
@@ -417,7 +417,7 @@ containerResources:
 #### Mellanox OFED driver
 
 | Name                                                        | Type   | Default                           | Description                                                                                                                                                                   |
-|-------------------------------------------------------------|--------|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ----------------------------------------------------------- | ------ | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `ofedDriver.deploy`                                         | bool   | `false`                           | deploy Mellanox OFED driver container                                                                                                                                         |
 | `ofedDriver.repository`                                     | string | `mellanox`                        | Mellanox OFED driver image repository                                                                                                                                         |
 | `ofedDriver.image`                                          | string | `mofed`                           | Mellanox OFED driver image name                                                                                                                                               |
@@ -448,6 +448,7 @@ containerResources:
 | `ofedDriver.upgradePolicy.waitForCompletion.podSelector`    | string | not set                           | specifies a label selector for the pods to wait for completion before starting the driver upgrade                                                                             |
 | `ofedDriver.upgradePolicy.waitForCompletion.timeoutSeconds` | int    | not set                           | specify the length of time in seconds to wait before giving up for workload to finish, zero means infinite                                                                    |
 | `ofedDriver.containerResources`                             | []     | not set                           | Optional [resource requests and limits](#container-resources) for the `mofed-container` container                                                                             |
+| `ofedDriver.forcePrecompiled`                               | bool   | `false`                           | Fail Mellanox OFED deployment if precompiled OFED driver container image does not exists                                                                                      |
 
 #### RDMA Device Plugin
 

diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
@@ -668,6 +668,13 @@ spec:
                       - name
                       type: object
                     type: array
+                  forcePrecompiled:
+                    default: false
+                    description: |-
+                      ForcePrecompiled specifies if only MOFED precompiled images are allowed
+                      If set to false and precompiled image does not exists, MOFED drivers will be compiled on Nodes
+                      If set to true and precompiled image does not exists, OFED state will be Error.
+                    type: boolean
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
                     type: string

diff --git a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml
@@ -32,6 +32,7 @@ spec:
     image: {{ .Values.ofedDriver.image }}
     repository: {{ .Values.ofedDriver.repository }}
     version: {{ .Values.ofedDriver.version }}
+    forcePrecompiled: {{ .Values.ofedDriver.forcePrecompiled }}
     {{- if .Values.ofedDriver.env }}
     env:
       {{ toYaml .Values.ofedDriver.env | nindent 6 }}

diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml
@@ -280,6 +280,7 @@ ofedDriver:
       # podSelector: "app=myapp"
       # specify the length of time in seconds to wait before giving up for workload to finish, zero means infinite
       # timeoutSeconds: 300
+  forcePrecompiled: false
 
 rdmaSharedDevicePlugin:
   deploy: true

diff --git a/go.mod b/go.mod
@@ -8,6 +8,8 @@ require (
 	github.com/caarlos0/env/v6 v6.10.1
 	github.com/containers/image/v5 v5.30.0
 	github.com/go-logr/logr v1.4.1
+	github.com/google/go-containerregistry v0.19.0
+	github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20231129213221-4fdaa32ee934
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.6.0
 	github.com/onsi/ginkgo/v2 v2.16.0
 	github.com/onsi/gomega v1.31.1
@@ -29,8 +31,13 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/containers/storage v1.53.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/cli v25.0.3+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v25.0.3+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.8.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
@@ -58,9 +65,11 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
@@ -70,16 +79,19 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect