feat: add option to start hosts as a DaemonSet

Add a `daemonset` option for managing hosts as a DaemonSet instead of just a Deployment. Signed-off-by: Dan Norris <[email protected]>
wasmCloud · Mar 26, 2024 · c9a881a · c9a881a
1 parent 6e56a7e
commit c9a881a
Show file tree

Hide file tree

Showing 7 changed files with 126 additions and 61 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "wasmcloud-operator"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [[bin]]

diff --git a/README.md b/README.md
@@ -19,7 +19,9 @@ spec:
   hostReplicas: 2
   # The cluster issuers to use for each host
   issuers:
-    - CDKF6OKPOBQKAX57UOXO7SCHURTOZWKWIVPC2HFJTGFXY5VJX44ECEHH
+    # This needs to be the public key of a cluster seed generated by
+    # `wash keys gen cluster`
+    - CDK...
   # The lattice to connect the hosts to
   lattice: 83a5b52e-17cf-4080-bac8-f844099f142e
   # Additional labels to apply to the host other than the defaults set in the operator
@@ -29,6 +31,8 @@ spec:
   version: 0.81.0
   # The name of a secret in the same namespace that provides the required secrets.
   secretName: cluster-secrets
+  # Enable the follwoing to run the wasmCloud hosts as a DaemonSet
+  #daemonset: true
 ```
 
 The CRD requires a Kubernetes Secret with the following keys:
@@ -40,7 +44,7 @@ metadata:
   name: my-wasmcloud-cluster
 data:
   # You can generate this with wash:
-  # wash keys gen cluster
+  # `wash keys gen cluster`
   WASMCLOUD_CLUSTER_SEED: <seed>
   # Only required if using a NATS creds file
   # nats.creds: <base64 encoded creds file>

diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "wasmcloud-operator-types"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [dependencies]

diff --git a/crates/types/src/v1alpha1/wasmcloud_host_config.rs b/crates/types/src/v1alpha1/wasmcloud_host_config.rs
@@ -17,6 +17,7 @@ use std::collections::HashMap;
 )]
 #[serde(rename_all = "camelCase")]
 pub struct WasmCloudHostConfigSpec {
+    #[serde(default = "default_host_replicas")]
     pub host_replicas: u32,
     pub issuers: Vec<String>,
     pub lattice: String,
@@ -37,6 +38,12 @@ pub struct WasmCloudHostConfigSpec {
     pub jetstream_domain: String,
     #[serde(default = "default_log_level")]
     pub log_level: String,
+    #[serde(default)]
+    pub daemonset: bool,
+}
+
+fn default_host_replicas() -> u32 {
+    1
 }
 
 fn default_jetstream_domain() -> String {

diff --git a/deploy/base/deployment.yaml b/deploy/base/deployment.yaml
@@ -61,6 +61,7 @@ rules:
       - apps
     resources:
       - deployments
+      - daemonsets
     verbs:
       - get
       - list

diff --git a/src/controller.rs b/src/controller.rs
@@ -3,7 +3,7 @@ use crate::{Error, Result};
 use anyhow::bail;
 use futures::StreamExt;
 use handlebars::Handlebars;
-use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec};
+use k8s_openapi::api::apps::v1::{DaemonSet, DaemonSetSpec, Deployment, DeploymentSpec};
 use k8s_openapi::api::core::v1::{
     ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar, EnvVarSource, ExecAction,
     Lifecycle, LifecycleHandler, PodSpec, PodTemplateSpec, Secret, SecretKeySelector,
@@ -148,7 +148,7 @@ async fn reconcile_crd(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Resul
         return Err(e);
     };
 
-    if let Err(e) = configure_deployment(&cfg, ctx.clone()).await {
+    if let Err(e) = configure_hosts(&cfg, ctx.clone()).await {
         warn!("Failed to configure deployment: {}", e);
         return Err(e);
     };
@@ -210,14 +210,10 @@ async fn cleanup(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Result<Acti
     Ok(Action::await_change())
 }
 
-fn deployment_spec(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> DeploymentSpec {
+fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplateSpec {
     let mut labels = BTreeMap::new();
     labels.insert("app.kubernetes.io/name".to_string(), config.name_any());
     labels.insert("app.kubernetes.io/instance".to_string(), config.name_any());
-    let ls = LabelSelector {
-        match_labels: Some(labels.clone()),
-        ..Default::default()
-    };
 
     let mut wasmcloud_env = vec![
         EnvVar {
@@ -375,45 +371,74 @@ fn deployment_spec(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> Deployme
             ..Default::default()
         },
     ];
+    PodTemplateSpec {
+        metadata: Some(ObjectMeta {
+            labels: Some(labels),
+            ..Default::default()
+        }),
+        spec: Some(PodSpec {
+            service_account: Some(config.name_any()),
+            containers,
+            volumes: Some(vec![
+                Volume {
+                    name: "nats-config".to_string(),
+                    config_map: Some(ConfigMapVolumeSource {
+                        name: Some(config.name_any()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                Volume {
+                    name: "nats-creds".to_string(),
+                    secret: Some(SecretVolumeSource {
+                        secret_name: Some(config.spec.secret_name.clone()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        }),
+    }
+}
+
+fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> DeploymentSpec {
+    let pod_template = pod_template(config, ctx);
+    let mut labels = BTreeMap::new();
+    labels.insert("app.kubernetes.io/name".to_string(), config.name_any());
+    labels.insert("app.kubernetes.io/instance".to_string(), config.name_any());
+    let ls = LabelSelector {
+        match_labels: Some(labels.clone()),
+        ..Default::default()
+    };
+
     DeploymentSpec {
         replicas: Some(config.spec.host_replicas as i32),
         selector: ls,
-        template: PodTemplateSpec {
-            metadata: Some(ObjectMeta {
-                labels: Some(labels),
-                ..Default::default()
-            }),
-            spec: Some(PodSpec {
-                service_account: Some(config.name_any()),
-                containers,
-                volumes: Some(vec![
-                    Volume {
-                        name: "nats-config".to_string(),
-                        config_map: Some(ConfigMapVolumeSource {
-                            name: Some(config.name_any()),
-                            ..Default::default()
-                        }),
-                        ..Default::default()
-                    },
-                    Volume {
-                        name: "nats-creds".to_string(),
-                        secret: Some(SecretVolumeSource {
-                            secret_name: Some(config.spec.secret_name.clone()),
-                            ..Default::default()
-                        }),
-                        ..Default::default()
-                    },
-                ]),
-                ..Default::default()
-            }),
-        },
+        template: pod_template,
         ..Default::default()
     }
 }
 
-async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Result<()> {
-    let mut spec = deployment_spec(config, ctx.clone());
+fn daemonset_spec(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> DaemonSetSpec {
+    let pod_template = pod_template(config, ctx);
+    let mut labels = BTreeMap::new();
+    labels.insert("app.kubernetes.io/name".to_string(), config.name_any());
+    labels.insert("app.kubernetes.io/instance".to_string(), config.name_any());
+    let ls = LabelSelector {
+        match_labels: Some(labels.clone()),
+        ..Default::default()
+    };
+
+    DaemonSetSpec {
+        selector: ls,
+        template: pod_template,
+        ..Default::default()
+    }
+}
 
+async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Result<()> {
+    let mut env_vars = vec![];
     if let Some(registry_credentials) = &config.spec.registry_credentials_secret {
         let secrets_client =
             Api::<Secret>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
@@ -443,7 +468,7 @@ async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc<Context>) -
             )));
         }
 
-        let mut env_vars = vec![
+        env_vars = vec![
             EnvVar {
                 name: "OCI_REGISTRY_USER".to_string(),
                 value: Some(
@@ -477,32 +502,60 @@ async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc<Context>) -
                 ..Default::default()
             },
         ];
+    }
 
+    if config.spec.daemonset {
+        let mut spec = daemonset_spec(config, ctx.clone());
         spec.template.spec.as_mut().unwrap().containers[1]
             .env
             .as_mut()
             .unwrap()
             .append(&mut env_vars);
-    }
+        let ds = DaemonSet {
+            metadata: ObjectMeta {
+                name: Some(config.name_any()),
+                namespace: Some(config.namespace().unwrap()),
+                owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+                ..Default::default()
+            },
+            spec: Some(spec),
+            ..Default::default()
+        };
 
-    let deployment = Deployment {
-        metadata: ObjectMeta {
-            name: Some(config.name_any()),
-            namespace: Some(config.namespace().unwrap()),
-            owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+        let api = Api::<DaemonSet>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
+        api.patch(
+            &config.name_any(),
+            &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
+            &Patch::Apply(ds),
+        )
+        .await?;
+    } else {
+        let mut spec = deployment_spec(config, ctx.clone());
+        spec.template.spec.as_mut().unwrap().containers[1]
+            .env
+            .as_mut()
+            .unwrap()
+            .append(&mut env_vars);
+        let deployment = Deployment {
+            metadata: ObjectMeta {
+                name: Some(config.name_any()),
+                namespace: Some(config.namespace().unwrap()),
+                owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+                ..Default::default()
+            },
+            spec: Some(spec),
             ..Default::default()
-        },
-        spec: Some(spec),
-        ..Default::default()
+        };
+
+        let api = Api::<Deployment>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
+        api.patch(
+            &config.name_any(),
+            &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
+            &Patch::Apply(deployment),
+        )
+        .await?;
     };
 
-    let api = Api::<Deployment>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
-    api.patch(
-        &config.name_any(),
-        &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
-        &Patch::Apply(deployment),
-    )
-    .await?;
     Ok(())
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -61,6 +61,7 @@ rules: @@
           - apps
         resources:
           - deployments
+          - daemonsets
         verbs:
           - get
           - list
@@ Expand Down @@