From 17aa8514c5a17bfc746c48d10b3da186a1430d2b Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Fri, 19 Feb 2021 18:11:21 -0700 Subject: [PATCH 1/8] fix Makefile for new yq version 4 --- examples/k8s/Makefile | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/k8s/Makefile b/examples/k8s/Makefile index d04a2614219..ae613e80ee1 100644 --- a/examples/k8s/Makefile +++ b/examples/k8s/Makefile @@ -1,3 +1,9 @@ +# NOTES: +# * This makefile has been switched to us yq v4, which has a syntax change. If +# you are seeing yq related errors, it may be that you are using yq v3. There +# are probably better ways to do this than with yq at this point, maybe +# kustomize. + .DEFAULT_GOAL := help .PHONY: help build build-and-push start stop show destroy destroy-all logs logs-master logs-slicer logs-worker teraslice-master setup setup-all rebuild elasticsearch deelasticsearch namespace delete-namespace role delete-role binding delete-binding auth deauth showauth SHELL := bash @@ -61,7 +67,8 @@ logs-worker: ## show logs for k8s teraslice workers master-start: ## start teraslice master in k8s or locally ifeq ($(TERASLICE_MODE),minikube) # Deploy master to Kubernetes - yq w masterDeployment.yaml spec.template.spec.containers[0].image $(TERASLICE_K8S_IMAGE) | kubectl create --namespace $(NAMESPACE) -f - + yq eval "select(documentIndex == 0) | .spec.template.spec.containers[0].image = \"$(TERASLICE_K8S_IMAGE)\"" masterDeployment.yaml | kubectl create --namespace $(NAMESPACE) -f - + yq eval "select(documentIndex == 1)" masterDeployment.yaml | kubectl create --namespace $(NAMESPACE) -f - else # Run Master Locally ./run-ts-master.sh @@ -102,19 +109,19 @@ deelasticsearch: elasticsearchDeployment.yaml ## delete elasticsearch from k8s kubectl delete --namespace $(NAMESPACE) -f ./elasticsearchDeployment.yaml namespace: ns.yaml ## create namespace - yq w ns.yaml metadata.name $(NAMESPACE) | kubectl create -f - + yq eval ".metadata.name = \"$(NAMESPACE)\"" ns.yaml | kubectl create -f - delete-namespace: - kubectl delete namespace $(NAMESPACE) + kubectl delete namespace $(NAMESPACE) || echo "* it is okay..." role: role.yaml ## create role with specified NAMESPACE - yq w role.yaml metadata.namespace $(NAMESPACE) | yq w - metadata.name teraslice-all-$(NAMESPACE) | kubectl create -f - + yq eval ".metadata.namespace = \"$(NAMESPACE)\"" role.yaml | yq eval ".metadata.name = \"teraslice-all-$(NAMESPACE)\"" - | kubectl create -f - delete-role: - kubectl delete --namespace $(NAMESPACE) roles teraslice-all-$(NAMESPACE) + kubectl delete --namespace $(NAMESPACE) roles teraslice-all-$(NAMESPACE) || echo "* it is okay..." binding: roleBinding.yaml ## bind NAMESPACE default service acount to teraslice-all-NAMESPACE role - yq w roleBinding.yaml metadata.namespace $(NAMESPACE) | yq w - metadata.name teraslice-all-$(NAMESPACE) | yq w - subjects[0].namespace $(NAMESPACE) | yq w - roleRef.name teraslice-all-$(NAMESPACE) | kubectl create -f - || true + yq eval ".metadata.namespace = \"$(NAMESPACE)\"" roleBinding.yaml | yq eval ".metadata.name = \"teraslice-all-$(NAMESPACE)\"" - | yq eval ".subjects[0].namespace = \"$(NAMESPACE)\"" - | yq eval ".roleRef.name = \"teraslice-all-$(NAMESPACE)\"" - | kubectl create -f - || true delete-binding: kubectl delete --namespace $(NAMESPACE) roleBindings teraslice-all-$(NAMESPACE) @@ -129,22 +136,22 @@ showauth: ## Show roles and roleBindings configs: ## create the configmaps ifeq ($(TERASLICE_MODE),minikube) - yq w teraslice-worker.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml + yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml kubectl create --namespace $(NAMESPACE) configmap teraslice-worker --from-file=teraslice.yaml || echo "* it is okay..." rm teraslice.yaml - yq w teraslice-master.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml + yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml kubectl create --namespace $(NAMESPACE) configmap teraslice-master --from-file=teraslice.yaml rm teraslice.yaml else # This adds the minikube vm's host machines IP where the master runs in the # hybrid case - yq w teraslice-worker.yaml.tpl teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.master_hostname $(HOST_IP) | yq w - teraslice.kubernetes_namespace $(NAMESPACE) > teraslice.yaml + yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.master_hostname = \"$(HOST_IP)\"" - | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml kubectl create --namespace $(NAMESPACE) configmap teraslice-worker --from-file=teraslice.yaml || echo "* it is okay..." rm teraslice.yaml # FIXME: Figure out where to clean this up # Note that the ES Port Here is 30200 because that is the port the service # NodePort gets exposed on - yq w teraslice-master.yaml.tpl terafoundation.connectors.elasticsearch.default.host[0] $(shell minikube ip):30200 | yq w - teraslice.kubernetes_image $(TERASLICE_K8S_IMAGE) | yq w - teraslice.assets_directory /tmp/assets > teraslice-master-local.yaml + yq eval ".terafoundation.connectors.elasticsearch.default.host[0] = \"$(shell minikube ip):30200\"" teraslice-master.yaml.tpl | yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" - | yq eval ".teraslice.assets_directory = \"/tmp/assets\"" - > teraslice-master-local.yaml endif build: ## builds docker images From 813bb188c4ca0d622c4c51438d7a72b90e22fc54 Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Fri, 19 Feb 2021 18:11:56 -0700 Subject: [PATCH 2/8] add docs for ephemeral_storage --- docs/configuration/clustering-k8s.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md index 3f3ed583e00..b4fe607c01a 100644 --- a/docs/configuration/clustering-k8s.md +++ b/docs/configuration/clustering-k8s.md @@ -115,6 +115,18 @@ Jobs so that they can be targetted to specific parts of your k8s infrastructure. Support for Kubernetes based clustering adds additional properties to a Teraslice job definition. These are outlined below. +### Ephemeral Storage + +If your Teraslice job uses a processor that needs temporary local storage that +persists in the Kubernets Pod across container restarts, you can set the +Teraslice Job Property `ephemeral_storage` to `true` on your job as shown +below. This will create an [`emptyDir`](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) +style Ephemeral Volume accessible in your pod at the path `/scratch`. + +```json + "ephemeral_storage": true +``` + ### Labels Key value pairs added into a job's `labels` array, as shown below, will result From ce5b58417741ff9af236a53bace4b1a80b65812c Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Sat, 20 Feb 2021 10:22:44 -0700 Subject: [PATCH 3/8] load standard asset during register --- examples/k8s/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/k8s/Makefile b/examples/k8s/Makefile index ae613e80ee1..357002e7e8d 100644 --- a/examples/k8s/Makefile +++ b/examples/k8s/Makefile @@ -185,6 +185,7 @@ rebuild: destroy build setup ## destroys then re-runs things make register register: ## creates asset and registers job + earl assets deploy ${TERASLICE_ALIAS} --blocking terascope/standard-assets earl assets deploy ${TERASLICE_ALIAS} --blocking terascope/elasticsearch-assets earl assets deploy ${TERASLICE_ALIAS} --blocking --build --replace --src-dir asset/ earl tjm register ${TERASLICE_ALIAS} example-job.json From 2279ca3bd1115c73bd689c735b30c956a2de2b77 Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Sat, 20 Feb 2021 12:26:08 -0700 Subject: [PATCH 4/8] add ephemeral_storage job option --- examples/k8s/example-job.json | 1 + packages/job-components/src/interfaces/context.ts | 2 ++ packages/job-components/src/interfaces/jobs.ts | 2 ++ packages/job-components/src/job-schemas.ts | 6 ++++++ .../cluster/backends/kubernetes/k8sResource.js | 14 ++++++++++++++ packages/teraslice/lib/config/schemas/system.js | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/examples/k8s/example-job.json b/examples/k8s/example-job.json index dd4b49cf24b..a49169487d7 100644 --- a/examples/k8s/example-job.json +++ b/examples/k8s/example-job.json @@ -7,6 +7,7 @@ "elasticsearch", "standard" ], + "ephemeral_storage": true, "operations": [ { "_op": "data_generator", diff --git a/packages/job-components/src/interfaces/context.ts b/packages/job-components/src/interfaces/context.ts index 639373c6f72..4fd0176e3b7 100644 --- a/packages/job-components/src/interfaces/context.ts +++ b/packages/job-components/src/interfaces/context.ts @@ -27,6 +27,8 @@ export interface TerasliceConfig { cpu?: number; /** This will only be available in the context of k8s */ cpu_execution_controller?: number|0.5; + /** This will only be available in the context of k8s */ + ephemeral_storage?: boolean|false; execution_controller_targets?: ExecutionControllerTargets[]; hostname: string; index_rollover_frequency: IndexRolloverFrequency; diff --git a/packages/job-components/src/interfaces/jobs.ts b/packages/job-components/src/interfaces/jobs.ts index b7fc0f1e7e1..ea32855556c 100644 --- a/packages/job-components/src/interfaces/jobs.ts +++ b/packages/job-components/src/interfaces/jobs.ts @@ -104,6 +104,8 @@ export interface ValidatedJobConfig { /** This will only be available in the context of k8s */ cpu_execution_controller?: number; /** This will only be available in the context of k8s */ + ephemeral_storage?: boolean; + /** This will only be available in the context of k8s */ memory?: number; /** This will only be available in the context of k8s */ memory_execution_controller?: number; diff --git a/packages/job-components/src/job-schemas.ts b/packages/job-components/src/job-schemas.ts index cd231be8a00..318ed1bfc75 100644 --- a/packages/job-components/src/job-schemas.ts +++ b/packages/job-components/src/job-schemas.ts @@ -232,6 +232,12 @@ export function jobSchema(context: Context): convict.Schema { format: 'Number', }; + schemas.ephemeral_storage = { + doc: 'Add ephemeral storage volume to worker and execution controller pods', + default: false, + format: Boolean + }; + schemas.memory = { doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes', default: undefined, diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js index 1a65bc57b09..67e28fc9537 100644 --- a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js +++ b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js @@ -50,6 +50,7 @@ class K8sResource { this._setVolumes(); this._setAssetsVolume(); this._setImagePullSecret(); + this._setEphemeralStorage(); if (resourceName === 'worker') { this._setAntiAffinity(); @@ -166,6 +167,19 @@ class K8sResource { } } + _setEphemeralStorage() { + if (this.execution.ephemeral_storage) { + this.resource.spec.template.spec.containers[0].volumeMounts.push({ + name: 'scratch-volume', + mountPath: '/scratch' + }); + this.resource.spec.template.spec.volumes.push({ + name: 'scratch-volume', + emptyDir: {} + }); + } + } + _setImagePullSecret() { if (this.terasliceConfig.kubernetes_image_pull_secret) { this.resource.spec.template.spec.imagePullSecrets = [ diff --git a/packages/teraslice/lib/config/schemas/system.js b/packages/teraslice/lib/config/schemas/system.js index aa242537584..f01c95f572f 100644 --- a/packages/teraslice/lib/config/schemas/system.js +++ b/packages/teraslice/lib/config/schemas/system.js @@ -231,6 +231,11 @@ const schema = { default: 0.5, format: 'Number' }, + ephemeral_storage: { + doc: 'Add ephemeral storage volume to worker and execution controller pods', + default: false, + format: Boolean + }, memory: { doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes', default: undefined, From 2e706ce82938d89d108f7f40b4ffa8ff6c65b445 Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Sun, 21 Feb 2021 18:12:38 -0700 Subject: [PATCH 5/8] add test for scratch volume --- .../backends/kubernetes/k8sResource-spec.js | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js index 26a842b6986..4d91df0159f 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js @@ -355,6 +355,33 @@ describe('k8sResource', () => { limits: cpu: 1`)); }); + + it('has scratch volume when ephemeral_storage is set true on execution', () => { + execution.ephemeral_storage = true; + + const kr = new K8sResource( + 'deployments', 'worker', terasliceConfig, execution + ); + + expect(kr.resource.spec.template.spec.containers[0].volumeMounts) + .toEqual( + [ + { mountPath: '/app/config', name: 'config' }, + { name: 'scratch-volume', mountPath: '/scratch' } + ] + ); + }); + + it('does not have scratch volume when ephemeral_storage is set false on execution', () => { + execution.ephemeral_storage = false; + + const kr = new K8sResource( + 'deployments', 'worker', terasliceConfig, execution + ); + + expect(kr.resource.spec.template.spec.containers[0].volumeMounts) + .toEqual([{ mountPath: '/app/config', name: 'config' }]); + }); }); describe('worker deployments with targets', () => { From dd66d2dbe0709e44a08f6d3a898f0aa018ea7d40 Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Mon, 22 Feb 2021 07:43:32 -0700 Subject: [PATCH 6/8] release: (minor) teraslice@0.73.0 --- package.json | 2 +- packages/teraslice/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4c6976889a5..df946d97761 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "0.72.1", + "version": "0.73.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 03cbc08ee20..01fb60cc150 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "0.72.1", + "version": "0.73.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { From 9d496b102412eb65192177af95609c875c2c0db1 Mon Sep 17 00:00:00 2001 From: peterdemartini Date: Mon, 22 Feb 2021 08:53:51 -0700 Subject: [PATCH 7/8] bump: (minor) @terascope/job-components@0.47.0, teraslice-client-js@0.35.0 bump: (minor) teraslice-cli@0.38.0 --- packages/job-components/package.json | 2 +- packages/teraslice-cli/package.json | 4 ++-- packages/teraslice-client-js/package.json | 4 ++-- packages/teraslice-op-test-harness/package.json | 4 ++-- packages/teraslice-test-harness/package.json | 4 ++-- packages/teraslice/package.json | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 8681f108f57..8b6cf27e2b5 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "0.46.2", + "version": "0.47.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 8b7f874ac90..4d0f7e37059 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "0.37.3", + "version": "0.38.0", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" @@ -48,7 +48,7 @@ "pretty-bytes": "^5.5.0", "prompts": "^2.4.0", "signale": "^1.4.0", - "teraslice-client-js": "^0.34.2", + "teraslice-client-js": "^0.35.0", "tmp": "^0.2.0", "tty-table": "^4.1.3", "yargs": "^16.2.0", diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index 9a458baec7e..a74c702c68b 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "0.34.2", + "version": "0.35.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -31,7 +31,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/job-components": "^0.46.2", + "@terascope/job-components": "^0.47.0", "auto-bind": "^4.0.0", "got": "^11.8.1" }, diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json index c0107ac9fa9..82be86918fc 100644 --- a/packages/teraslice-op-test-harness/package.json +++ b/packages/teraslice-op-test-harness/package.json @@ -21,10 +21,10 @@ "bluebird": "^3.7.2" }, "devDependencies": { - "@terascope/job-components": "^0.46.2" + "@terascope/job-components": "^0.47.0" }, "peerDependencies": { - "@terascope/job-components": "^0.46.2" + "@terascope/job-components": "^0.47.0" }, "publishConfig": { "access": "public", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index 8369ef65ec1..7f3a26f26b1 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "^9.1.0" }, "devDependencies": { - "@terascope/job-components": "^0.46.2" + "@terascope/job-components": "^0.47.0" }, "peerDependencies": { - "@terascope/job-components": "^0.46.2" + "@terascope/job-components": "^0.47.0" }, "engines": { "node": ">=10.16.0" diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 01fb60cc150..ed36611b6b6 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -38,7 +38,7 @@ }, "dependencies": { "@terascope/elasticsearch-api": "^2.18.2", - "@terascope/job-components": "^0.46.2", + "@terascope/job-components": "^0.47.0", "@terascope/teraslice-messaging": "^0.20.2", "@terascope/utils": "^0.36.2", "async-mutex": "^0.3.0", From 76a03a5d5fa8e98f3485f1b9d45889f8af2d4130 Mon Sep 17 00:00:00 2001 From: peterdemartini Date: Mon, 22 Feb 2021 09:30:44 -0700 Subject: [PATCH 8/8] Rename scratch directory to ephemeral0 --- docs/configuration/clustering-k8s.md | 22 +++++++++---------- .../backends/kubernetes/k8sResource.js | 6 ++--- .../backends/kubernetes/k8sResource-spec.js | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md index b4fe607c01a..15cf4b78035 100644 --- a/docs/configuration/clustering-k8s.md +++ b/docs/configuration/clustering-k8s.md @@ -92,17 +92,17 @@ it's just kind of nice to have. The table below shows the Teraslice Master configuration settings added to support k8s based Teraslice deployments. -| Configuration | Description | Type | Notes | -| :----------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------: | :----: | :------: | -| assets_volume | Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored | String | optional | -| cpu_execution_controller | CPU resources to use for Execution Controller request and limit values | Number | optional | -| execution_controller_targets | array of `{"key": "rack", "value": "alpha"}` targets for execution controllers | String | optional | -| kubernetes_image | Name of docker image, default: `teraslice:k8sdev` | String | optional | -| kubernetes_image_pull_secret | Secret used to pull docker images from private repo | String | optional | -| kubernetes_config_map_name | Name of the configmap used by worker and execution_controller containers for config. If this is not provided, the default will be `-worker` | String | optional | -| kubernetes_namespace | Kubernetes Namespace that Teraslice will run in, default namespace: 'default' | String | optional | +| Configuration | Description | Type | Notes | +| :----------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----: | :------: | +| assets_volume | Name of kubernetes volume to be shared across all pods, where Teraslice assets will be stored | String | optional | +| cpu_execution_controller | CPU resources to use for Execution Controller request and limit values | Number | optional | +| execution_controller_targets | array of `{"key": "rack", "value": "alpha"}` targets for execution controllers | String | optional | +| kubernetes_image | Name of docker image, default: `teraslice:k8sdev` | String | optional | +| kubernetes_image_pull_secret | Secret used to pull docker images from private repo | String | optional | +| kubernetes_config_map_name | Name of the configmap used by worker and execution_controller containers for config. If this is not provided, the default will be `-worker` | String | optional | +| kubernetes_namespace | Kubernetes Namespace that Teraslice will run in, default namespace: 'default' | String | optional | | kubernetes_worker_antiaffinity | If `true`, pod antiaffinity will be enabled for Teraslice workers, `false` by default | Boolean | optional | -| memory_execution_controller | Memory resources to use for Execution Controller request and limit values | Number | optional | +| memory_execution_controller | Memory resources to use for Execution Controller request and limit values | Number | optional | Note that the `assets_volume` should also be mounted to your Teraslice master pod. @@ -121,7 +121,7 @@ If your Teraslice job uses a processor that needs temporary local storage that persists in the Kubernets Pod across container restarts, you can set the Teraslice Job Property `ephemeral_storage` to `true` on your job as shown below. This will create an [`emptyDir`](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) -style Ephemeral Volume accessible in your pod at the path `/scratch`. +style Ephemeral Volume accessible in your pod at the path `/ephemeral0`. ```json "ephemeral_storage": true diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js index 67e28fc9537..775c52f3d07 100644 --- a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js +++ b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js @@ -170,11 +170,11 @@ class K8sResource { _setEphemeralStorage() { if (this.execution.ephemeral_storage) { this.resource.spec.template.spec.containers[0].volumeMounts.push({ - name: 'scratch-volume', - mountPath: '/scratch' + name: 'ephemeral-volume', + mountPath: '/ephemeral0' }); this.resource.spec.template.spec.volumes.push({ - name: 'scratch-volume', + name: 'ephemeral-volume', emptyDir: {} }); } diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js index 4d91df0159f..b8e8fad3b9e 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js @@ -367,7 +367,7 @@ describe('k8sResource', () => { .toEqual( [ { mountPath: '/app/config', name: 'config' }, - { name: 'scratch-volume', mountPath: '/scratch' } + { name: 'ephemeral-volume', mountPath: '/ephemeral0' } ] ); });