diff --git a/.travis.yml b/.travis.yml index d9fb84cd8bb..e0ee2f7cb24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ dist: bionic os: linux language: node_js -node_js: '12.19' +node_js: '12.20' # use a smaller depth for faster builds git: depth: 10 @@ -48,31 +48,33 @@ jobs: script: yarn --silent test --suite unit-a - name: 'Unit Test Suite (node 12)' - node_js: '12.19' + node_js: '12.20' # run only on pull-requests or cron if: branch = master AND type IN (pull_request, cron) script: yarn --silent test --suite unit-b - script: name: 'ES Test Suite (elasticsearch 6) (node 12)' - node_js: '12.19' + node_js: '12.20' # run only on pull-requests and cron if: branch = master AND type IN (pull_request, cron) script: yarn --silent test --suite elasticsearch --elasticsearch-version 6.8.6 --elasticsearch-api-version 6.5 - script: name: 'ES Test Suite (elasticsearch 7) (node 12)' - node_js: '12.19' + node_js: '12.20' # run only on pull-requests if: branch = master AND type IN (pull_request) AND commit_message !~ /^WIP/ script: yarn --silent test --suite elasticsearch --elasticsearch-version 7.2.1 --elasticsearch-api-version 7.0 --report-coverage false - script: name: 'End-to-End Test Suite (elasticsearch 6) (node 12)' - node_js: '12.19' + node_js: '12.20' # run only on pull-requests and cron if: branch = master AND type IN (pull_request, cron) AND fork = false - script: yarn --silent --cwd e2e test + script: + - export SERVICE_HEAP_OPTS="-Xms768m -Xmx768m" + - yarn --silent --cwd e2e test - stage: "Releases" name: 'Publish prerelease packages and docker image' @@ -84,7 +86,7 @@ jobs: - script: name: 'Publish packages, docs and expiremental docker image' - node_js: '12.19' + node_js: '12.20' # run a push to master if: tag IS blank AND branch = master AND type NOT IN (pull_request, cron) script: diff --git a/docs/asset-bundles/development.md b/docs/asset-bundles/development.md index c9617454e26..b4e7666411c 100644 --- a/docs/asset-bundles/development.md +++ b/docs/asset-bundles/development.md @@ -56,23 +56,20 @@ operator just like any other operator, as shown below: "name": "Update Rate Test", "lifecycle": "once", "workers": 1, - "assets": ["elasticsearch", "example"], + "assets": ["elasticsearch", "standard", "example"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000 }, { "_op": "count", "log_level": "debug" }, - { - "_op": "elasticsearch_index_selector", - "index": "update-test-1", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "update-test-1", + "type": "events", "size": 5000 } ] diff --git a/docs/jobs/builtin-operations.md b/docs/jobs/builtin-operations.md index e12c2073f7a..c7866eada66 100644 --- a/docs/jobs/builtin-operations.md +++ b/docs/jobs/builtin-operations.md @@ -35,10 +35,10 @@ Example Job: `examples/jobs/script/test_script_job.json` "name": "ES DataGen test script", "lifecycle": "persistent", "workers": 1, - "assets": ["elasticsearch"], + "assets": ["standard"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 100000, "stress_test": true }, diff --git a/docs/jobs/configuration.md b/docs/jobs/configuration.md index 42c547dd1b3..5ae32b8190e 100644 --- a/docs/jobs/configuration.md +++ b/docs/jobs/configuration.md @@ -32,7 +32,7 @@ The first operation in the [operations](#operations) list, reads from a particul | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | | `_op` | Name of operation, it must reflect the exact name of the file | `String` | required | | `_encoding` | Used for specifying the data encoding type when using `DataEntity.fromBuffer`. Defaults to `json`. | `String` | optional | -| `_dead_letter_action` | This action will specify what to do when failing to parse or transform a record. ​​​​​The following builtin actions are supported, "throw", "log", or "none". If none of the actions are specified it will try and use a registered [Dead Letter Queue](./dead-letter-queue.md) API under that name. The API must be already be created by a operation before it can used.​ | `String` | required | +| `_dead_letter_action` | This action will specify what to do when failing to parse or transform a record. ​​​​​The following builtin actions are supported, "throw", "log", or "none". If none of the actions are specified it will try and use a registered [Dead Letter Queue](./dead-letter-queue.md) API under that name. The API must be already be created by a operation before it can used.​ | `String` | required | ### apis @@ -62,13 +62,10 @@ The first operation in the [operations](#operations) list, reads from a particul "_op": "custom_op", "some": "configuration" }, - { - "_op": "elasticsearch_index_selector", - "index": "bigdata3", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "bigdata3", + "type": "events", "size": 5000 } ] diff --git a/docs/packages/teraslice-client-js/overview.md b/docs/packages/teraslice-client-js/overview.md index d7ade0c776a..4bbee2f4141 100644 --- a/docs/packages/teraslice-client-js/overview.md +++ b/docs/packages/teraslice-client-js/overview.md @@ -27,18 +27,16 @@ var job = { "name": "Data Generator", "lifecycle": "once", "workers": 1, + "assets": ["standard", "elasticsearch"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 1 }, - { - "_op": "elasticsearch_index_selector", - "index": "client-test-logs", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "client-test-logs", + "type": "events", "size": 50 } ] diff --git a/e2e/test/cases/assets/simple-spec.js b/e2e/test/cases/assets/simple-spec.js index 7fcada1fcfc..a643ce6a094 100644 --- a/e2e/test/cases/assets/simple-spec.js +++ b/e2e/test/cases/assets/simple-spec.js @@ -29,7 +29,7 @@ describe('assets', () => { const result = await teraslice.assets.upload(fileStream, { blocking: true }); // NOTE: In this case, the asset is referenced by the ID // assigned by teraslice and not it's name. - jobSpec.assets = [result._id, 'elasticsearch']; + jobSpec.assets = [result._id, 'standard', 'elasticsearch']; const ex = await submitAndStart(jobSpec); @@ -116,7 +116,7 @@ describe('assets', () => { it('can directly ask for the new asset to be used', async () => { const jobSpec = misc.newJob('generator-asset'); - jobSpec.assets = ['ex1:0.1.1', 'elasticsearch']; + jobSpec.assets = ['ex1:0.1.1', 'standard', 'elasticsearch']; const { workers } = jobSpec; const assetResponse = await teraslice.assets.getAsset('ex1', '0.1.1'); diff --git a/e2e/test/download-assets.js b/e2e/test/download-assets.js index f26c55e89d6..547068953cd 100644 --- a/e2e/test/download-assets.js +++ b/e2e/test/download-assets.js @@ -19,6 +19,10 @@ const bundles = [ { repo: 'kafka-assets', name: 'kafka' + }, + { + repo: 'standard-assets', + name: 'standard' } ]; @@ -42,7 +46,7 @@ function getOlderAsset(assets, assetName) { } function filterRelease(release) { - return !release.draft && !release.prerelease; + return !release.draft; } function filterAsset(asset) { diff --git a/e2e/test/fixtures/jobs/generate-to-es.json b/e2e/test/fixtures/jobs/generate-to-es.json index eb0f46f1007..d8301baa243 100644 --- a/e2e/test/fixtures/jobs/generate-to-es.json +++ b/e2e/test/fixtures/jobs/generate-to-es.json @@ -4,20 +4,17 @@ "lifecycle": "persistent", "workers": 2, "analytics": false, - "assets": ["elasticsearch"], + "assets": ["elasticsearch", "standard"], "max_retries": 0, "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 100 }, - { - "_op": "elasticsearch_index_selector", - "index": "replace-me-1000", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "replace-me-1000", + "type": "events", "size": 1000 } ] diff --git a/e2e/test/fixtures/jobs/generator-asset.json b/e2e/test/fixtures/jobs/generator-asset.json index 3cabb70475e..e0cbf88bdfc 100644 --- a/e2e/test/fixtures/jobs/generator-asset.json +++ b/e2e/test/fixtures/jobs/generator-asset.json @@ -3,12 +3,12 @@ "slicers": 1, "lifecycle": "persistent", "workers": 2, - "assets": ["ex1", "elasticsearch"], + "assets": ["ex1", "standard"], "max_retries": 0, "analytics": false, "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 1000 }, { diff --git a/e2e/test/fixtures/jobs/generator.json b/e2e/test/fixtures/jobs/generator.json index 9e21b1417d4..28bcf98d2c1 100644 --- a/e2e/test/fixtures/jobs/generator.json +++ b/e2e/test/fixtures/jobs/generator.json @@ -4,11 +4,11 @@ "lifecycle": "persistent", "workers": 3, "analytics": false, - "assets": ["elasticsearch"], + "assets": ["standard"], "max_retries": 0, "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 1000, "stress_test": false }, diff --git a/e2e/test/fixtures/jobs/id.json b/e2e/test/fixtures/jobs/id.json index 26cf899cb12..844482d35e3 100644 --- a/e2e/test/fixtures/jobs/id.json +++ b/e2e/test/fixtures/jobs/id.json @@ -13,13 +13,10 @@ "size": 500, "key_type": "base64url" }, - { - "_op": "elasticsearch_index_selector", - "index": "test-id_reindex-1000", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "test-id_reindex-1000", + "type": "events", "size": 200 } ] diff --git a/e2e/test/fixtures/jobs/kafka-reader.json b/e2e/test/fixtures/jobs/kafka-reader.json index 9eb5d776fd0..f1eeb469ceb 100644 --- a/e2e/test/fixtures/jobs/kafka-reader.json +++ b/e2e/test/fixtures/jobs/kafka-reader.json @@ -16,13 +16,10 @@ "_encoding": "json" }, { - "_op": "elasticsearch_index_selector", + "_op": "elasticsearch_bulk", "type": "events", "index": "replace-me-1000", - "preserve_id": true - }, - { - "_op": "elasticsearch_bulk", + "preserve_id": true, "size": 500 } ] diff --git a/e2e/test/fixtures/jobs/multisend.json b/e2e/test/fixtures/jobs/multisend.json index 1668852a481..52be220596b 100644 --- a/e2e/test/fixtures/jobs/multisend.json +++ b/e2e/test/fixtures/jobs/multisend.json @@ -14,13 +14,10 @@ "size": 500 }, { - "_op": "elasticsearch_index_selector", + "_op": "elasticsearch_bulk", "index": "multisend-1000", "preserve_id": true, - "type": "change" - }, - { - "_op": "elasticsearch_bulk", + "type": "change", "multisend_index_append": false, "size": 100, "multisend": true, diff --git a/e2e/test/fixtures/jobs/reindex.json b/e2e/test/fixtures/jobs/reindex.json index 1aa791e014c..36b40ff7e75 100644 --- a/e2e/test/fixtures/jobs/reindex.json +++ b/e2e/test/fixtures/jobs/reindex.json @@ -14,13 +14,10 @@ "preserve_id": true }, { - "_op": "elasticsearch_index_selector", + "_op": "elasticsearch_bulk", "type": "change", "index": "replace-me-1000", - "preserve_id": true - }, - { - "_op": "elasticsearch_bulk", + "preserve_id": true, "size": 50 } ] diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 435ef0cc0f9..12e8887a42b 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -65,19 +65,16 @@ async function generateTestData() { name: `Generate: ${indexName}`, lifecycle: 'once', workers: 1, - assets: ['elasticsearch'], + assets: ['elasticsearch', 'standard'], operations: [ { - _op: 'elasticsearch_data_generator', + _op: 'data_generator', size: count }, - { - _op: 'elasticsearch_index_selector', - index: indexName, - type: 'events' - }, { _op: 'elasticsearch_bulk', + index: indexName, + type: 'events', size: 1000 } ] @@ -107,7 +104,7 @@ async function generateTestData() { } try { - await Promise.all(misc.EXAMLPE_INDEX_SIZES.map((size) => generate(size))); + await Promise.all(misc.EXAMPLE_INDEX_SIZES.map((size) => generate(size))); // we need fully active jobs so we can get proper meta data for recovery state tests signale.success('Data generation is done', getElapsed(startTime)); } catch (err) { diff --git a/e2e/test/misc.js b/e2e/test/misc.js index caa1a9daf4c..bfdb0fe3467 100644 --- a/e2e/test/misc.js +++ b/e2e/test/misc.js @@ -28,7 +28,7 @@ const CONFIG_PATH = path.join(BASE_PATH, '.config'); const ASSETS_PATH = path.join(BASE_PATH, '.assets'); const SPEC_INDEX_PREFIX = `${TEST_INDEX_PREFIX}spec`; const EXAMPLE_INDEX_PREFIX = `${TEST_INDEX_PREFIX}example`; -const EXAMLPE_INDEX_SIZES = [100, 1000]; +const EXAMPLE_INDEX_SIZES = [100, 1000]; // the uniq cluster name const CLUSTER_NAME = newId(`${TEST_INDEX_PREFIX}teracluster`, true, 2); @@ -38,7 +38,7 @@ const DEFAULT_WORKERS = 2; // The teraslice-master + the number of teraslice-worker instances (see the docker-compose.yml) const DEFAULT_NODES = DEFAULT_WORKERS + 1; // The number of workers per number (see the process-master.yaml and process-worker.yaml) -const WORKERS_PER_NODE = 12; +const WORKERS_PER_NODE = 8; const compose = new Compose('docker-compose.yml'); const signale = require('./signale'); @@ -76,7 +76,7 @@ function newSpecIndex(name) { } function getExampleIndex(size) { - if (!EXAMLPE_INDEX_SIZES.includes(size)) { + if (!EXAMPLE_INDEX_SIZES.includes(size)) { throw new Error(`No example index with ${size}`); } @@ -187,7 +187,7 @@ module.exports = { globalTeardown, newSpecIndex, resetLogs, - EXAMLPE_INDEX_SIZES, + EXAMPLE_INDEX_SIZES, EXAMPLE_INDEX_PREFIX, SPEC_INDEX_PREFIX, ELASTICSEARCH_HOST, diff --git a/examples/jobs/data_generator.json b/examples/jobs/data_generator.json index af0766ea3f6..8ce092c43c6 100644 --- a/examples/jobs/data_generator.json +++ b/examples/jobs/data_generator.json @@ -2,20 +2,17 @@ "name": "Data Generator", "lifecycle": "persistent", "workers": 1, + "assets": ["elasticsearch", "standard"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000 }, - { - "_op": "elasticsearch_index_selector", - "index": "example-logs", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "example-logs", + "type": "events", "size": 5000 } ] } - diff --git a/examples/jobs/hitcounter.json b/examples/jobs/hitcounter.json index 9545890fba2..182aa77e7c7 100644 --- a/examples/jobs/hitcounter.json +++ b/examples/jobs/hitcounter.json @@ -17,14 +17,11 @@ { "_op": "summarize" }, - { - "_op": "elasticsearch_index_selector", - "type": "change", - "index": "hitcounts", - "id_field": "_key" - }, { "_op": "elasticsearch_bulk", + "type": "change", + "index": "hitcounts", + "id_field": "_key", "size": 10000 } ] diff --git a/examples/jobs/reindex.json b/examples/jobs/reindex.json index 08db5f7e78c..187c0c0c24b 100644 --- a/examples/jobs/reindex.json +++ b/examples/jobs/reindex.json @@ -13,13 +13,10 @@ "preserve_id": true }, { - "_op": "elasticsearch_index_selector", + "_op": "elasticsearch_bulk", "type": "change", "index": "example-logs-new", - "preserve_id": true - }, - { - "_op": "elasticsearch_bulk", + "preserve_id": true, "size": 10000 } ] diff --git a/examples/jobs/script/test_script_job.json b/examples/jobs/script/test_script_job.json index cef7e09a358..5cd34e446ee 100644 --- a/examples/jobs/script/test_script_job.json +++ b/examples/jobs/script/test_script_job.json @@ -2,9 +2,10 @@ "name": "ES DataGen test script", "lifecycle": "persistent", "workers": 1, + "assets": ["standard"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 100000, "stress_test": true }, diff --git a/examples/k8s/example-job-labels.json b/examples/k8s/example-job-labels.json index 3f66a6132b8..f1b691b0402 100644 --- a/examples/k8s/example-job-labels.json +++ b/examples/k8s/example-job-labels.json @@ -4,14 +4,15 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "labels": { "key1": "value1" }, "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { @@ -19,9 +20,9 @@ }, { "_op": "elasticsearch_bulk", - "size": 5000, "index": "terak8s-example-data", - "type": "events" + "type": "events", + "size": 5000 } ] } diff --git a/examples/k8s/example-job-resource.json b/examples/k8s/example-job-resource.json index e7786d03961..a375a5688ce 100644 --- a/examples/k8s/example-job-resource.json +++ b/examples/k8s/example-job-resource.json @@ -4,13 +4,14 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "cpu": 1, "memory": 536870912, "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { @@ -18,9 +19,9 @@ }, { "_op": "elasticsearch_bulk", - "size": 5000, "index": "terak8s-example-data", - "type": "events" + "type": "events", + "size": 5000 } ] } diff --git a/examples/k8s/example-job-targets.json b/examples/k8s/example-job-targets.json index 16e07dc4814..202f064f13c 100644 --- a/examples/k8s/example-job-targets.json +++ b/examples/k8s/example-job-targets.json @@ -4,7 +4,8 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "targets": [ { @@ -14,7 +15,7 @@ ], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { @@ -22,9 +23,9 @@ }, { "_op": "elasticsearch_bulk", - "size": 5000, "index": "terak8s-example-data", - "type": "events" + "type": "events", + "size": 5000 } ] } diff --git a/examples/k8s/example-job-volume.json b/examples/k8s/example-job-volume.json index 9bfb5fa68a7..c9abb6625b5 100644 --- a/examples/k8s/example-job-volume.json +++ b/examples/k8s/example-job-volume.json @@ -4,7 +4,8 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "volumes": [ { @@ -14,7 +15,7 @@ ], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { @@ -22,9 +23,9 @@ }, { "_op": "elasticsearch_bulk", - "size": 5000, "index": "terak8s-example-data", - "type": "events" + "type": "events", + "size": 5000 } ] } diff --git a/examples/k8s/example-job-with-image.json b/examples/k8s/example-job-with-image.json index c84af7c20a3..d521e46516c 100644 --- a/examples/k8s/example-job-with-image.json +++ b/examples/k8s/example-job-with-image.json @@ -5,11 +5,12 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { diff --git a/examples/k8s/example-job.json b/examples/k8s/example-job.json index 2ae12f37082..dd4b49cf24b 100644 --- a/examples/k8s/example-job.json +++ b/examples/k8s/example-job.json @@ -4,11 +4,12 @@ "workers": 2, "assets": [ "example", - "elasticsearch" + "elasticsearch", + "standard" ], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 5000000 }, { @@ -20,9 +21,9 @@ }, { "_op": "elasticsearch_bulk", - "size": 5000, "index": "terak8s-example-data", - "type": "events" + "type": "events", + "size": 5000 } ] } diff --git a/examples/k8s/sample-job.json b/examples/k8s/sample-job.json index 615828646df..84ef6711848 100644 --- a/examples/k8s/sample-job.json +++ b/examples/k8s/sample-job.json @@ -2,9 +2,10 @@ "name": "gen-tenk-noop", "lifecycle": "persistent", "workers": 1, + "assets": ["standard"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "json_schema": "", "size": 10000, "start": "", diff --git a/package.json b/package.json index c393a7f291e..1c66ee9e204 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "0.72.0", + "version": "0.72.1", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 8873b0b38fc..d7840a8988a 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "0.37.2", + "version": "0.37.3", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" diff --git a/packages/teraslice-cli/src/cmds/tjm/init.ts b/packages/teraslice-cli/src/cmds/tjm/init.ts index eb32435071f..db6915c0f4c 100644 --- a/packages/teraslice-cli/src/cmds/tjm/init.ts +++ b/packages/teraslice-cli/src/cmds/tjm/init.ts @@ -22,21 +22,19 @@ export = { name: 'data-generator', lifecycle: 'persistent', workers: 3, + assets: ['elasticsearch', 'standard'], operations: [ { - _op: 'elasticsearch_data_generator', + _op: 'data_generator', size: 5000 }, - { - _op: 'elasticsearch_index_selector', - index: 'example-logs', - type: 'events' - }, { _op: 'elasticsearch_bulk', size: 5000, - connection: 'default' - }] + index: 'example-logs', + type: 'events' + } + ] }; job.validateJob(); job.overwrite(); diff --git a/packages/teraslice-cli/src/helpers/asset-src.ts b/packages/teraslice-cli/src/helpers/asset-src.ts index 8ac1e723836..4fb02f83836 100644 --- a/packages/teraslice-cli/src/helpers/asset-src.ts +++ b/packages/teraslice-cli/src/helpers/asset-src.ts @@ -128,9 +128,15 @@ export class AssetSrc { } // run npm --cwd srcDir/asset --prod --silent --no-progress - reply.info('* running yarn --prod --no-progress'); + reply.info('* running yarn --prod --no-progress'); await this._yarnCmd(path.join(tmpDir.name, 'asset'), ['--prod', '--no-progress']); + // run yarn --cwd srcDir --prod --silent --no-progress asset:post-build + if (this.packageJson?.scripts && this.packageJson.scripts['asset:post-build']) { + reply.info('* running yarn asset:post-build'); + await this._yarnCmd(tmpDir.name, ['run', 'asset:post-build']); + } + try { reply.info('* zipping the asset bundle'); // create zipfile @@ -154,7 +160,7 @@ export class AssetSrc { throw new Error(`Missing asset directory "${tmpAssetDir}"`); } - await execa('zip', ['--symlinks', '-q', '-r', '-9', outputFileName, '.'], { + await execa('zip', ['-q', '-r', '-9', outputFileName, '.'], { stdio: 'inherit', cwd: tmpAssetDir }); diff --git a/packages/teraslice-cli/test/fixtures/cliJobFile.json b/packages/teraslice-cli/test/fixtures/cliJobFile.json index bffe5015c37..5d1994ed411 100644 --- a/packages/teraslice-cli/test/fixtures/cliJobFile.json +++ b/packages/teraslice-cli/test/fixtures/cliJobFile.json @@ -2,19 +2,16 @@ "name": "Data Generator", "lifecycle": "once", "workers": 1, - "assets": ["elasticsearch"], + "assets": ["elasticsearch", "standard"], "operations": [ { - "_op": "elasticsearch_data_generator", + "_op": "data_generator", "size": 10000 }, - { - "_op": "elasticsearch_index_selector", - "index": "auto-data-logs", - "type": "events" - }, { "_op": "elasticsearch_bulk", + "index": "auto-data-logs", + "type": "events", "size": 100 } ], diff --git a/packages/teraslice-client-js/examples/submit.js b/packages/teraslice-client-js/examples/submit.js index d5b5aa2616b..3a162019d51 100644 --- a/packages/teraslice-client-js/examples/submit.js +++ b/packages/teraslice-client-js/examples/submit.js @@ -12,19 +12,16 @@ const exampleJob = { name: 'Data Generator', lifecycle: 'once', workers: 1, - assets: ['elasticsearch'], + assets: ['elasticsearch', 'standard'], operations: [ { - _op: 'elasticsearch_data_generator', + _op: 'data_generator', size: 1 }, - { - _op: 'elasticsearch_index_selector', - index: 'client-test-logs', - type: 'events' - }, { _op: 'elasticsearch_bulk', + index: 'client-test-logs', + type: 'events', size: 50 } ] diff --git a/packages/teraslice/lib/storage/assets.js b/packages/teraslice/lib/storage/assets.js index 2191b539fef..44861acbee8 100644 --- a/packages/teraslice/lib/storage/assets.js +++ b/packages/teraslice/lib/storage/assets.js @@ -4,14 +4,14 @@ const path = require('path'); const fse = require('fs-extra'); const crypto = require('crypto'); const { - TSError, pDelay, uniq, isString, toString, filterObject + TSError, uniq, isString, toString, filterObject } = require('@terascope/utils'); const elasticsearchBackend = require('./backends/elasticsearch_store'); const { makeLogger } = require('../workers/helpers/terafoundation'); const { saveAsset } = require('../utils/file_utils'); const { findMatchingAsset, findSimilarAssets, - toVersionQuery, getInCompatiblityReason + toVersionQuery, getInCompatibilityReason } = require('../utils/asset_utils'); // Module to manager job states in Elasticsearch. @@ -71,7 +71,7 @@ module.exports = async function assetsStore(context) { id, data, esData, blocking }) { const startTime = Date.now(); - const metaData = await saveAsset(logger, assetsPath, id, data, _metaIsUnqiue); + const metaData = await saveAsset(logger, assetsPath, id, data, _metaIsUnique); const assetRecord = Object.assign({ blob: esData, @@ -148,7 +148,7 @@ module.exports = async function assetsStore(context) { const found = findMatchingAsset(assets, name, version); if (!found) { - const reason = getInCompatiblityReason(findSimilarAssets(assets, name, version), ', due to a potential'); + const reason = getInCompatibilityReason(findSimilarAssets(assets, name, version), ', due to a potential'); throw new TSError(`No asset found for "${assetIdentifier}"${reason}`, { statusCode: 404 }); @@ -160,7 +160,7 @@ module.exports = async function assetsStore(context) { return Promise.all(uniq(assetsArray).map(_getAssetId)); } - async function _metaIsUnqiue(meta) { + async function _metaIsUnique(meta) { const includes = ['name', 'version', 'node_version', 'platform', 'arch']; const query = Object.entries(filterObject(meta, { includes })) @@ -230,13 +230,11 @@ module.exports = async function assetsStore(context) { const assets = await findAssetsToAutoload(autoloadDir); if (!assets || !assets.length) return; - const promises = assets.map(async (asset, i) => { - await pDelay(i * 100); - + for (const asset of assets) { logger.info(`autoloading asset ${asset}...`); const assetPath = path.join(autoloadDir, asset); try { - const result = await save(await fse.readFile(assetPath)); + const result = await save(await fse.readFile(assetPath), true); if (result.created) { logger.debug(`autoloaded asset ${asset}`); } else { @@ -249,9 +247,7 @@ module.exports = async function assetsStore(context) { throw err; } } - }); - - await Promise.all(promises); + } logger.info('done autoloading assets'); } diff --git a/packages/teraslice/lib/utils/asset_utils.js b/packages/teraslice/lib/utils/asset_utils.js index 376c954b8b6..b8be22c66be 100644 --- a/packages/teraslice/lib/utils/asset_utils.js +++ b/packages/teraslice/lib/utils/asset_utils.js @@ -23,8 +23,8 @@ function findSimilarAssets(records, name, version) { return assets; } -function getInCompatiblityReason(assets, prefix) { - if (!assets || !assets.length) return null; +function getInCompatibilityReason(assets, prefix) { + if (!assets || !assets.length) return ''; const reasons = []; @@ -40,7 +40,7 @@ function getInCompatiblityReason(assets, prefix) { } }); - if (!reasons.length) return null; + if (!reasons.length) return ''; return `${prefix ? `${trim(prefix)} ` : ''}${joinList(reasons, ',', 'or')} mismatch`; } @@ -107,7 +107,7 @@ function toVersionQuery(_version) { module.exports = { findSimilarAssets, - getInCompatiblityReason, + getInCompatibilityReason, getMajorVersion, findMatchingAsset, toSemverRange, diff --git a/packages/teraslice/lib/utils/file_utils.js b/packages/teraslice/lib/utils/file_utils.js index b1b460320b7..bc270bbd294 100644 --- a/packages/teraslice/lib/utils/file_utils.js +++ b/packages/teraslice/lib/utils/file_utils.js @@ -4,10 +4,13 @@ const fs = require('fs'); const path = require('path'); const fse = require('fs-extra'); const semver = require('semver'); +const { Mutex } = require('async-mutex'); const { TSError } = require('@terascope/utils'); const decompress = require('decompress'); const { getMajorVersion } = require('./asset_utils'); +const mutex = new Mutex(); + function existsSync(filename) { try { fs.accessSync(filename); @@ -72,7 +75,7 @@ async function verifyAssetJSON(id, newPath) { } } -async function saveAsset(logger, assetsPath, id, binaryData, metaCheck) { +async function _saveAsset(logger, assetsPath, id, binaryData, metaCheck) { const newPath = path.join(assetsPath, id); try { @@ -101,6 +104,12 @@ async function saveAsset(logger, assetsPath, id, binaryData, metaCheck) { } } +async function saveAsset(logger, assetsPath, id, binaryData, metaCheck) { + return mutex.runExclusive(() => _saveAsset( + logger, assetsPath, id, binaryData, metaCheck + )); +} + module.exports = { existsSync, saveAsset, diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 76d762d1146..bce0af46818 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "0.72.0", + "version": "0.72.1", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { diff --git a/packages/teraslice/test/utils/asset_utils-spec.js b/packages/teraslice/test/utils/asset_utils-spec.js index e11a51c15e8..46730c2e7fc 100644 --- a/packages/teraslice/test/utils/asset_utils-spec.js +++ b/packages/teraslice/test/utils/asset_utils-spec.js @@ -3,7 +3,7 @@ const { toSemverRange, findMatchingAsset, getMajorVersion, toVersionQuery, - findSimilarAssets, getInCompatiblityReason, + findSimilarAssets, getInCompatibilityReason, } = require('../../lib/utils/asset_utils'); describe('Asset Utils', () => { @@ -123,15 +123,15 @@ describe('Asset Utils', () => { }); }); - describe('->getInCompatiblityReason/->findSimilarAssets', () => { + describe('->getInCompatibilityReason/->findSimilarAssets', () => { test.each([ ['foo', 'latest', 'node_version or platform mismatch'], ['foo', '2.0.1', 'platform mismatch'], ['foo', '~2.0.0', 'node_version or platform mismatch'], - ['foo', '3.*', null], - ['foo', '0.1.*', null], + ['foo', '3.*', ''], + ['foo', '0.1.*', ''], ])('should return the correct result for %s:%s', (name, version, result) => { - const reason = getInCompatiblityReason(findSimilarAssets(assets, name, version)); + const reason = getInCompatibilityReason(findSimilarAssets(assets, name, version)); expect(reason).toEqual(result); }); });