Merge pull request #3348 from terascope/k8s-update-Mar-2023

Adds `pod_spec_override` to the job
Adds `kubernetes_overrides_enabled` to Teraslice config

docs/configuration/clustering-k8s.md

@@ -95,15 +95,16 @@ The table below shows the Teraslice Master configuration settings added to
 support k8s based Teraslice deployments.
 
 |         Configuration          |                                                                        Description                                                                         |  Type   |  Notes   |
-| :----------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----: | :------: |
+|:------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------:|:-------:|:--------:|
 |         assets_volume          |                               Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored                                | String  | optional |
 |    cpu_execution_controller    |                                           CPU resources to use for Execution Controller request and limit values                                           | Number  | optional |
 |  execution_controller_targets  |                                       array of `{"key": "rack", "value": "alpha"}` targets for execution controllers                                       | String  | optional |
 |        kubernetes_image        |                                                     Name of docker image, default: `teraslice:k8sdev`                                                      | String  | optional |
 |  kubernetes_image_pull_secret  |                                                    Secret used to pull docker images from private repo                                                     | String  | optional |
 |   kubernetes_config_map_name   | Name of the configmap used by worker and execution_controller containers for config.  If this is not provided, the default will be `<CLUSTER_NAME>-worker` | String  | optional |
 |      kubernetes_namespace      |                                       Kubernetes Namespace that Teraslice will run in, default namespace: 'default'                                        | String  | optional |
-| kubernetes_priority_class_name |                       Priority class that the Teraslice master, execution controller, and stateful workers should run with                                 | String  | optional |
+|  kubernetes_overrides_enabled  |                                       Enable the `pod_spec_override` feature on job definitions, `false` by default.                                       | Boolean | optional |
+| kubernetes_priority_class_name |                            Priority class that the Teraslice master, execution controller, and stateful workers should run with                            | String  | optional |
 | kubernetes_worker_antiaffinity |                                   If `true`, pod antiaffinity will be enabled for Teraslice workers, `false` by default                                    | Boolean | optional |
 |  memory_execution_controller   |                                         Memory resources to use for Execution Controller request and limit values                                          | Number  | optional |
 
@@ -422,7 +423,7 @@ targets:
           effect: NoSchedule
 ```
 
-## Attach existing volumes
+### Attach existing volumes
 
 One or more volumes can be specified on your job and these volumes will be
 attached to your worker and execution controller pods at runtime.  The volumes
@@ -436,6 +437,39 @@ property should be the name of the Kubernetes `persistentVolumeClaim`.  The
 ],
 ```
 
+### Pod `.spec` Overrides
+
+Teraslice provides the `pod_spec_override` job property which allows the user to
+specify any arbitrary Kubernetes configuration to be overlaid on the
+[pod](https://kubernetes.io/docs/concepts/workloads/pods/) `.spec` for both the
+execution controller and worker pods.  This allows users to "sneak" in
+functionality that we have not yet explicitly implemented.
+
+For instance, this feature can be used to add `initContainers` or `hostAliases`.
+A `pod_spec_override` must be in the form of JSON that maps directly to the
+appropriate YAML for the Kubernetes resource.  An example of a
+`pod_spec_override` that could be added to a job is shown below:
+
+```json
+    "pod_spec_override": {
+        "initContainers": [
+            {
+                "name": "init-hello-world",
+                "image": "busybox:1.28",
+                "command": [
+                    "sh",
+                    "-c",
+                    "echo 'HELLO WORLD'"
+                ]
+            }
+        ]
+    },
+```
+
+**NOTE:** In certain circumstances, this feature could be abused by a malicious
+job author, so it is off by default.  It can be enabled by setting
+`kubernetes_overrides_enabled: true`.
+
 ## Kubernetes Labels
 
 We automatically add labels to the Kubernetes resources created by Teraslice.
@@ -475,7 +509,6 @@ export NAMESPACE=ts-dev1
 export TERASLICE_K8S_IMAGE=teraslice-k8sdev:1
 export TERASLICE_MODE=minikube
 minikube start --memory 4096 --cpus 4
-eval $(minikube docker-env)
 make build
 make setup-all
 make show

examples/k8s/Makefile

@@ -145,10 +145,10 @@ showpriority:  ## Show Kubernetes Priority Class
 
 configs: ## create the configmaps
 ifeq ($(TERASLICE_MODE),minikube)
-	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
+	yq eval ".teraslice.kubernetes_image = \"docker.io/library/$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
 	kubectl create --namespace $(NAMESPACE) configmap teraslice-worker --from-file=teraslice.yaml || echo "* it is okay..."
 	rm teraslice.yaml
-	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
+	yq eval ".teraslice.kubernetes_image = \"docker.io/library/$(TERASLICE_K8S_IMAGE)\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
 	kubectl create --namespace $(NAMESPACE) configmap teraslice-master --from-file=teraslice.yaml
 	rm teraslice.yaml
 else
@@ -164,7 +164,7 @@ else
 endif
 
 build: ## builds docker images
-	docker build -t $(TERASLICE_K8S_IMAGE) ../..
+	minikube image build -t $(TERASLICE_K8S_IMAGE) ../..
 
 push: ## push final docker image
 	docker push $(TERASLICE_K8S_IMAGE)
@@ -184,7 +184,7 @@ destroy: deregister ## delete k8s deployments and services
 	curl -fsS -XDELETE "$(ES_URL)/terak8s-example-data" || echo '* it is okay'
 
 destroy-img: destroy
-	docker rmi $(shell docker image ls $(TERASLICE_K8S_IMAGE) --format "{{.ID}}") || echo "* it is okay..."
+	minikube image rm docker.io/library/$(TERASLICE_K8S_IMAGE) || echo "* it is okay..."
 
 destroy-all: destroy-img deauth deelasticsearch delete-namespace master-stop ## delete ALL things including the namespace
 	earl aliases remove ${TERASLICE_ALIAS} || echo '* it is okay'
@@ -205,6 +205,7 @@ register: ## creates asset and registers job
 	earl tjm register ${TERASLICE_ALIAS} example-job-stateful.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-targets.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-volume.json
+	earl tjm register ${TERASLICE_ALIAS} example-job-with-pod-spec-override.json
 
 deregister: ## resets jobs
 	earl tjm reset example-job.json || echo '* it is okay'
@@ -214,6 +215,7 @@ deregister: ## resets jobs
 	earl tjm reset example-job-stateful.json || echo '* it is okay'
 	earl tjm reset example-job-targets.json || echo '* it is okay'
 	earl tjm reset example-job-volume.json || echo '* it is okay'
+	earl tjm reset example-job-with-pod-spec-override.json || echo '* it is okay'
 
 start: ## starts example job
 	# yes | tjm asset --replace -c $(TERASLICE_MASTER_URL) || echo '* it is okay'

examples/k8s/example-job-with-pod-spec-override.json

@@ -0,0 +1,43 @@
+{
+    "name": "example-data-generator-job-with-override",
+    "lifecycle": "once",
+    "workers": 2,
+    "assets": [
+        "example",
+        "elasticsearch",
+        "standard"
+    ],
+    "ephemeral_storage": true,
+    "operations": [
+        {
+            "_op": "data_generator",
+            "size": 5000000
+        },
+        {
+            "_op": "example-op"
+        },
+        {
+            "_op": "delay",
+            "ms": 30000
+        },
+        {
+            "_op": "elasticsearch_bulk",
+            "index": "terak8s-example-data",
+            "type": "events",
+            "size": 5000
+        }
+    ],
+    "pod_spec_override": {
+        "initContainers": [
+            {
+                "name": "init-hello-world",
+                "image": "busybox:1.28",
+                "command": [
+                    "sh",
+                    "-c",
+                    "echo 'HELLO WORLD'"
+                ]
+            }
+        ]
+    }
+}

examples/k8s/teraslice-master.yaml.tpl

@@ -24,6 +24,7 @@ teraslice:
     kubernetes_image_pull_secrets:
         - "docker-tera1-secret"
     kubernetes_namespace: "ts-dev1"
+    kubernetes_overrides_enabled: true
     kubernetes_priority_class_name: 'high-priority'
     name: "ts-dev1"
     cpu: 1

examples/k8s/teraslice-worker.yaml.tpl

@@ -22,6 +22,7 @@ teraslice:
     master_hostname: "teraslice-master"
     kubernetes_image: "teraslice-k8sdev"
     kubernetes_namespace: "ts-dev1"
+    kubernetes_overrides_enabled: true
     kubernetes_priority_class_name: 'high-priority'
     name: "ts-dev1"
     cpu: 1

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.82.1",
+    "version": "0.83.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/job-components/src/interfaces/context.ts

@@ -37,6 +37,7 @@ export interface TerasliceConfig {
     kubernetes_image_pull_secret?: string|'';
     kubernetes_image?: string|'terascope/teraslice';
     kubernetes_namespace?: string|'default';
+    kubernetes_overrides_enabled?: boolean|false;
     kubernetes_priority_class_name?: string|'';
     kubernetes_worker_antiaffinity?: boolean|false;
     master_hostname: string|'localhost';

packages/job-components/src/interfaces/jobs.ts

@@ -113,6 +113,8 @@ export interface ValidatedJobConfig {
     /** This will only be available in the context of k8s */
     memory_execution_controller?: number;
     /** This will only be available in the context of k8s */
+    pod_spec_override?: AnyObject;
+    /** This will only be available in the context of k8s */
     resources_requests_cpu?: number;
     /** This will only be available in the context of k8s */
     resources_requests_memory?: number;

packages/job-components/src/job-schemas.ts

@@ -1,6 +1,7 @@
 import os from 'os';
 import convict from 'convict';
 import {
+    AnyObject,
     DataEncoding,
     dataEncodings,
     flatten,
@@ -308,6 +309,20 @@ export function jobSchema(context: Context): convict.Schema<any> {
             format: 'Number',
         };
 
+        schemas.pod_spec_override = {
+            doc: 'foo',
+            default: {},
+            format(obj: AnyObject) {
+                if (!isPlainObject(obj)) {
+                    throw new Error('must be object');
+                }
+                if ((Object.keys(obj).length !== 0)
+                    && (!context.sysconfig.teraslice.kubernetes_overrides_enabled)) {
+                    throw new Error('The Teraslice master must set \'kubernetes_overrides_enabled: true\' to use pod_spec_override in a job.');
+                }
+            }
+        };
+
         schemas.resources_requests_cpu = {
             doc: 'kubernetes CPU request, in cores, to set on Teraslice workers',
             default: undefined,

packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/index.js

@@ -94,7 +94,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
 
         execution.slicer_port = 45680;
         const exJobResource = new K8sResource(
-            'jobs', 'execution_controller', context.sysconfig.teraslice, execution
+            'jobs', 'execution_controller', context.sysconfig.teraslice, execution, logger
         );
         const exJob = exJobResource.resource;
 
@@ -135,7 +135,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
         execution.k8sUid = jobs.items[0].metadata.uid;
 
         const kr = new K8sResource(
-            'deployments', 'worker', context.sysconfig.teraslice, execution
+            'deployments', 'worker', context.sysconfig.teraslice, execution, logger
         );
 
         const workerDeployment = kr.resource;

packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -22,10 +22,11 @@ class K8sResource {
      * @param {Object} terasliceConfig - teraslice cluster config from context
      * @param {Object} execution - teraslice execution
      */
-    constructor(resourceType, resourceName, terasliceConfig, execution) {
+    constructor(resourceType, resourceName, terasliceConfig, execution, logger) {
         this.execution = execution;
         this.jobLabelPrefix = 'job.teraslice.terascope.io';
         this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
+        this.logger = logger;
         this.nodeType = resourceName;
         this.terasliceConfig = terasliceConfig;
 
@@ -68,6 +69,10 @@ class K8sResource {
         if (resourceName === 'execution_controller') {
             this._setExecutionControllerTargets();
         }
+
+        if (this.terasliceConfig.kubernetes_overrides_enabled) {
+            this._mergePodSpecOverlay();
+        }
     }
 
     _makeConfig() {
@@ -412,6 +417,27 @@ class K8sResource {
             effect: 'NoSchedule'
         });
     }
+
+    /**
+     * _mergePodSpecOverlay - allows the author of the job to override anything
+     * in the pod .spec for both the execution controller and the worker pods
+     * created in Kubernetes.  This can be useful in many ways including these:
+     *
+     *   * add `initContainers` to the pods
+     *   * add `hostAliases` to the pods
+     *
+     * Note that this happens at the end of the process, so anything added by
+     * this overlay will overwrite any other setting set on the job or by the
+     * config.
+     *
+     * Job setting: `pod_spec_override`
+     */
+    _mergePodSpecOverlay() {
+        this.resource.spec.template.spec = _.merge(
+            this.resource.spec.template.spec,
+            this.execution.pod_spec_override
+        );
+    }
 }
 
 module.exports = K8sResource;

packages/teraslice/lib/config/schemas/system.js

@@ -295,6 +295,11 @@ const schema = {
         default: 'default',
         format: 'optional_String'
     },
+    kubernetes_overrides_enabled: {
+        doc: '',
+        default: false,
+        format: Boolean
+    },
     kubernetes_priority_class_name: {
         doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
         default: undefined,

packages/teraslice/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.82.1",
+    "version": "0.83.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {

packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js

@@ -918,4 +918,24 @@ describe('k8sResource', () => {
             expect(krExporter.resource.spec.template.metadata.labels['job-property.teraslice.terascope.io/stateful']).toEqual('true');
         });
     });
+
+    describe('teraslice config with kubernetes_overrides_enabled set false', () => {
+        it('generates pods without overlay in pod .spec', () => {
+            execution.pod_spec_override = { initContainers: [] };
+            terasliceConfig.kubernetes_overrides_enabled = false;
+
+            const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution);
+            expect(krWorker.resource.spec.template.spec).not.toHaveProperty('initContainers');
+        });
+    });
+
+    describe('teraslice config with kubernetes_overrides_enabled set true', () => {
+        it('generates pods with overlay in pod .spec', () => {
+            execution.pod_spec_override = { initContainers: [] };
+            terasliceConfig.kubernetes_overrides_enabled = true;
+
+            const krWorker = new K8sResource('deployments', 'worker', terasliceConfig, execution);
+            expect(krWorker.resource.spec.template.spec).toHaveProperty('initContainers');
+        });
+    });
 });