Merge pull request #2507 from terascope/add-ephemeral-volume

Add ephemeral volume

docs/configuration/clustering-k8s.md

@@ -92,17 +92,17 @@ it's just kind of nice to have.
 The table below shows the Teraslice Master configuration settings added to
 support k8s based Teraslice deployments.
 
-|        Configuration           |                                                                        Description                                                                         |  Type  |  Notes   |
-| :----------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------: | :----: | :------: |
-|         assets_volume          |                               Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored                                | String | optional |
-|    cpu_execution_controller    |                                       CPU resources to use for Execution Controller request and limit values                                               | Number | optional |
-|  execution_controller_targets  |                                 array of `{"key": "rack", "value": "alpha"}` targets for execution controllers                                             | String | optional |
-|        kubernetes_image        |                                                     Name of docker image, default: `teraslice:k8sdev`                                                      | String | optional |
-|  kubernetes_image_pull_secret  |                                                    Secret used to pull docker images from private repo                                                     | String | optional |
-|   kubernetes_config_map_name   | Name of the configmap used by worker and execution_controller containers for config.  If this is not provided, the default will be `<CLUSTER_NAME>-worker` | String | optional |
-|      kubernetes_namespace      |                                       Kubernetes Namespace that Teraslice will run in, default namespace: 'default'                                        | String | optional |
+|         Configuration          |                                                                        Description                                                                         |  Type   |  Notes   |
+| :----------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----: | :------: |
+|         assets_volume          |                               Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored                                | String  | optional |
+|    cpu_execution_controller    |                                           CPU resources to use for Execution Controller request and limit values                                           | Number  | optional |
+|  execution_controller_targets  |                                       array of `{"key": "rack", "value": "alpha"}` targets for execution controllers                                       | String  | optional |
+|        kubernetes_image        |                                                     Name of docker image, default: `teraslice:k8sdev`                                                      | String  | optional |
+|  kubernetes_image_pull_secret  |                                                    Secret used to pull docker images from private repo                                                     | String  | optional |
+|   kubernetes_config_map_name   | Name of the configmap used by worker and execution_controller containers for config.  If this is not provided, the default will be `<CLUSTER_NAME>-worker` | String  | optional |
+|      kubernetes_namespace      |                                       Kubernetes Namespace that Teraslice will run in, default namespace: 'default'                                        | String  | optional |
 | kubernetes_worker_antiaffinity |                                   If `true`, pod antiaffinity will be enabled for Teraslice workers, `false` by default                                    | Boolean | optional |
-|   memory_execution_controller  |                                       Memory resources to use for Execution Controller request and limit values                                            | Number | optional |
+|  memory_execution_controller   |                                         Memory resources to use for Execution Controller request and limit values                                          | Number  | optional |
 
 Note that the `assets_volume` should also be mounted to your Teraslice master pod.
 
@@ -115,6 +115,18 @@ Jobs so that they can be targetted to specific parts of your k8s infrastructure.
 Support for Kubernetes based clustering adds additional properties to a
 Teraslice job definition.  These are outlined below.
 
+### Ephemeral Storage
+
+If your Teraslice job uses a processor that needs temporary local storage that
+persists in the Kubernets Pod across container restarts, you can set the
+Teraslice Job Property `ephemeral_storage` to `true` on your job as shown
+below.  This will create an [`emptyDir`](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir)
+style Ephemeral Volume accessible in your pod at the path `/ephemeral0`.
+
+```json
+    "ephemeral_storage": true
+```
+
 ### Labels
 
 Key value pairs added into a job's `labels` array, as shown below, will result

examples/k8s/Makefile

@@ -1,3 +1,9 @@
+# NOTES:
+# * This makefile has been switched to us yq v4, which has a syntax change.  If
+#   you are seeing yq related errors, it may be that you are using yq v3.  There
+#   are probably better ways to do this than with yq at this point, maybe
+#   kustomize.
+
 .DEFAULT_GOAL := help
 .PHONY: help build build-and-push start stop show destroy destroy-all logs logs-master logs-slicer logs-worker teraslice-master setup setup-all rebuild elasticsearch deelasticsearch namespace delete-namespace role delete-role binding delete-binding auth deauth showauth
 SHELL := bash
@@ -61,7 +67,8 @@ logs-worker: ## show logs for k8s teraslice workers
 master-start: ## start teraslice master in k8s or locally
 ifeq ($(TERASLICE_MODE),minikube)
 	# Deploy master to Kubernetes
-	yq w masterDeployment.yaml spec.template.spec.containers[0].image $(TERASLICE_K8S_IMAGE) | kubectl create --namespace $(NAMESPACE) -f -
+	yq eval "select(documentIndex == 0) | .spec.template.spec.containers[0].image = \"$(TERASLICE_K8S_IMAGE)\"" masterDeployment.yaml | kubectl create --namespace $(NAMESPACE) -f -
+	yq eval "select(documentIndex == 1)" masterDeployment.yaml | kubectl create --namespace $(NAMESPACE) -f -
 else
 	# Run Master Locally
 	./run-ts-master.sh
@@ -102,19 +109,19 @@ deelasticsearch: elasticsearchDeployment.yaml ## delete elasticsearch from k8s
 	kubectl delete --namespace $(NAMESPACE) -f ./elasticsearchDeployment.yaml
 
 namespace: ns.yaml ## create namespace
-	yq w ns.yaml metadata.name $(NAMESPACE) | kubectl create -f -
+	yq eval ".metadata.name = \"$(NAMESPACE)\"" ns.yaml | kubectl create -f -
 
 delete-namespace:
-	kubectl delete namespace $(NAMESPACE)
+	kubectl delete namespace $(NAMESPACE) || echo "* it is okay..."
 
 role: role.yaml ## create role with specified NAMESPACE
-	yq w role.yaml metadata.namespace $(NAMESPACE) | yq w - metadata.name teraslice-all-$(NAMESPACE) | kubectl create -f -
+	yq eval ".metadata.namespace = \"$(NAMESPACE)\"" role.yaml | yq eval ".metadata.name = \"teraslice-all-$(NAMESPACE)\"" - | kubectl create -f -
 
 delete-role:
-	kubectl delete --namespace $(NAMESPACE) roles teraslice-all-$(NAMESPACE)
+	kubectl delete --namespace $(NAMESPACE) roles teraslice-all-$(NAMESPACE) || echo "* it is okay..."
 
 binding: roleBinding.yaml ## bind NAMESPACE default service acount to teraslice-all-NAMESPACE role
-	yq w roleBinding.yaml metadata.namespace $(NAMESPACE) | yq w - metadata.name teraslice-all-$(NAMESPACE) | yq w - subjects[0].namespace $(NAMESPACE) | yq w - roleRef.name teraslice-all-$(NAMESPACE) | kubectl create -f - || true
+	yq eval ".metadata.namespace = \"$(NAMESPACE)\"" roleBinding.yaml | yq eval ".metadata.name = \"teraslice-all-$(NAMESPACE)\"" - | yq eval ".subjects[0].namespace = \"$(NAMESPACE)\"" - | yq eval ".roleRef.name = \"teraslice-all-$(NAMESPACE)\"" - | kubectl create -f - || true
 
 delete-binding:
 	kubectl delete --namespace $(NAMESPACE) roleBindings teraslice-all-$(NAMESPACE)
@@ -129,22 +136,22 @@ showauth: ## Show roles and roleBindings
 
 configs: ## create the configmaps
 ifeq ($(TERASLICE_MODE),minikube)
-	yq w teraslice-worker.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml
+	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
 	kubectl create --namespace $(NAMESPACE) configmap teraslice-worker --from-file=teraslice.yaml || echo "* it is okay..."
 	rm teraslice.yaml
-	yq w teraslice-master.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml
+	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
 	kubectl create --namespace $(NAMESPACE) configmap teraslice-master --from-file=teraslice.yaml
 	rm teraslice.yaml
 else
 	# This adds the minikube vm's host machines IP where the master runs in the
 	# hybrid case
-	yq w teraslice-worker.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.master_hostname $(HOST_IP) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml
+	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.master_hostname = \"$(HOST_IP)\"" - | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
 	kubectl create --namespace $(NAMESPACE) configmap teraslice-worker --from-file=teraslice.yaml || echo "* it is okay..."
 	rm teraslice.yaml
 	# FIXME: Figure out where to clean this up
 	# Note that the ES Port Here is 30200 because that is the port the service
 	# NodePort gets exposed on
-	yq w teraslice-master.yaml.tpl terafoundation.connectors.elasticsearch.default.host[0] $(shell minikube ip):30200 | yq w - teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.assets_directory /tmp/assets > teraslice-master-local.yaml
+	yq eval ".terafoundation.connectors.elasticsearch.default.host[0] = \"$(shell minikube ip):30200\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" - | yq eval ".teraslice.assets_directory = \"/tmp/assets\"" - > teraslice-master-local.yaml
 endif
 
 build: ## builds docker images
@@ -178,6 +185,7 @@ rebuild: destroy build setup ## destroys then re-runs things
 	make register
 
 register: ## creates asset and registers job
+	earl assets deploy ${TERASLICE_ALIAS} --blocking terascope/standard-assets
 	earl assets deploy ${TERASLICE_ALIAS} --blocking terascope/elasticsearch-assets
 	earl assets deploy ${TERASLICE_ALIAS} --blocking --build --replace --src-dir asset/
 	earl tjm register ${TERASLICE_ALIAS} example-job.json

examples/k8s/example-job.json

@@ -7,6 +7,7 @@
         "elasticsearch",
         "standard"
     ],
+    "ephemeral_storage": true,
     "operations": [
         {
             "_op": "data_generator",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.72.1",
+    "version": "0.73.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/job-components/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/job-components",
     "displayName": "Job Components",
-    "version": "0.46.2",
+    "version": "0.47.0",
     "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
     "bugs": {

packages/job-components/src/interfaces/context.ts

@@ -27,6 +27,8 @@ export interface TerasliceConfig {
     cpu?: number;
     /** This will only be available in the context of k8s */
     cpu_execution_controller?: number|0.5;
+    /** This will only be available in the context of k8s */
+    ephemeral_storage?: boolean|false;
     execution_controller_targets?: ExecutionControllerTargets[];
     hostname: string;
     index_rollover_frequency: IndexRolloverFrequency;

packages/job-components/src/interfaces/jobs.ts

@@ -104,6 +104,8 @@ export interface ValidatedJobConfig {
     /** This will only be available in the context of k8s */
     cpu_execution_controller?: number;
     /** This will only be available in the context of k8s */
+    ephemeral_storage?: boolean;
+    /** This will only be available in the context of k8s */
     memory?: number;
     /** This will only be available in the context of k8s */
     memory_execution_controller?: number;

packages/job-components/src/job-schemas.ts

@@ -232,6 +232,12 @@ export function jobSchema(context: Context): convict.Schema<any> {
             format: 'Number',
         };
 
+        schemas.ephemeral_storage = {
+            doc: 'Add ephemeral storage volume to worker and execution controller pods',
+            default: false,
+            format: Boolean
+        };
+
         schemas.memory = {
             doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes',
             default: undefined,

packages/teraslice-cli/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "0.37.3",
+    "version": "0.38.0",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"
@@ -48,7 +48,7 @@
         "pretty-bytes": "^5.5.0",
         "prompts": "^2.4.0",
         "signale": "^1.4.0",
-        "teraslice-client-js": "^0.34.2",
+        "teraslice-client-js": "^0.35.0",
         "tmp": "^0.2.0",
         "tty-table": "^4.1.3",
         "yargs": "^16.2.0",

packages/teraslice-client-js/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-client-js",
     "displayName": "Teraslice Client (JavaScript)",
-    "version": "0.34.2",
+    "version": "0.35.0",
     "description": "A Node.js client for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "elasticsearch",
@@ -31,7 +31,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/job-components": "^0.46.2",
+        "@terascope/job-components": "^0.47.0",
         "auto-bind": "^4.0.0",
         "got": "^11.8.1"
     },

packages/teraslice-op-test-harness/package.json

@@ -21,10 +21,10 @@
         "bluebird": "^3.7.2"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.46.2"
+        "@terascope/job-components": "^0.47.0"
     },
     "peerDependencies": {
-        "@terascope/job-components": "^0.46.2"
+        "@terascope/job-components": "^0.47.0"
     },
     "publishConfig": {
         "access": "public",

packages/teraslice-test-harness/package.json

@@ -36,10 +36,10 @@
         "fs-extra": "^9.1.0"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.46.2"
+        "@terascope/job-components": "^0.47.0"
     },
     "peerDependencies": {
-        "@terascope/job-components": "^0.46.2"
+        "@terascope/job-components": "^0.47.0"
     },
     "engines": {
         "node": ">=10.16.0"

packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -50,6 +50,7 @@ class K8sResource {
         this._setVolumes();
         this._setAssetsVolume();
         this._setImagePullSecret();
+        this._setEphemeralStorage();
 
         if (resourceName === 'worker') {
             this._setAntiAffinity();
@@ -166,6 +167,19 @@ class K8sResource {
         }
     }
 
+    _setEphemeralStorage() {
+        if (this.execution.ephemeral_storage) {
+            this.resource.spec.template.spec.containers[0].volumeMounts.push({
+                name: 'ephemeral-volume',
+                mountPath: '/ephemeral0'
+            });
+            this.resource.spec.template.spec.volumes.push({
+                name: 'ephemeral-volume',
+                emptyDir: {}
+            });
+        }
+    }
+
     _setImagePullSecret() {
         if (this.terasliceConfig.kubernetes_image_pull_secret) {
             this.resource.spec.template.spec.imagePullSecrets = [

packages/teraslice/lib/config/schemas/system.js

@@ -231,6 +231,11 @@ const schema = {
         default: 0.5,
         format: 'Number'
     },
+    ephemeral_storage: {
+        doc: 'Add ephemeral storage volume to worker and execution controller pods',
+        default: false,
+        format: Boolean
+    },
     memory: {
         doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes',
         default: undefined,

packages/teraslice/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.72.1",
+    "version": "0.73.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -38,7 +38,7 @@
     },
     "dependencies": {
         "@terascope/elasticsearch-api": "^2.18.2",
-        "@terascope/job-components": "^0.46.2",
+        "@terascope/job-components": "^0.47.0",
         "@terascope/teraslice-messaging": "^0.20.2",
         "@terascope/utils": "^0.36.2",
         "async-mutex": "^0.3.0",

packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js

@@ -355,6 +355,33 @@ describe('k8sResource', () => {
                   limits:
                     cpu: 1`));
         });
+
+        it('has scratch volume when ephemeral_storage is set true on execution', () => {
+            execution.ephemeral_storage = true;
+
+            const kr = new K8sResource(
+                'deployments', 'worker', terasliceConfig, execution
+            );
+
+            expect(kr.resource.spec.template.spec.containers[0].volumeMounts)
+                .toEqual(
+                    [
+                        { mountPath: '/app/config', name: 'config' },
+                        { name: 'ephemeral-volume', mountPath: '/ephemeral0' }
+                    ]
+                );
+        });
+
+        it('does not have scratch volume when ephemeral_storage is set false on execution', () => {
+            execution.ephemeral_storage = false;
+
+            const kr = new K8sResource(
+                'deployments', 'worker', terasliceConfig, execution
+            );
+
+            expect(kr.resource.spec.template.spec.containers[0].volumeMounts)
+                .toEqual([{ mountPath: '/app/config', name: 'config' }]);
+        });
     });
 
     describe('worker deployments with targets', () => {