diff --git a/docs/management-apis/endpoints-json.md b/docs/management-apis/endpoints-json.md index 81f7103e49e..f0113b02284 100644 --- a/docs/management-apis/endpoints-json.md +++ b/docs/management-apis/endpoints-json.md @@ -197,6 +197,7 @@ Returns an array of all jobs listed in `${clusterName}__jobs` index. **Query Options:** - `active: string = [true|false]` +- `deleted: string = [true|false]` - `from: number = 0` - `size: number = 100` - `sort: string = "_updated:desc"` @@ -205,7 +206,11 @@ Setting `active` to `true` will return only the jobs considered active, which includes the jobs that have `active` set to `true` as well as those that do not have an `active` property. If your query sets `active` to `false` it will only return the jobs with the `active` property set to false. If the `active` query -parameteris not provided, all jobs will be returned. +parameter is not provided, all jobs will be returned. + +Setting `deleted` to `false` or not setting the option will return jobs +where `_deleted` is set to `false` or the `_deleted` key is not present. +Setting `deleted` to `true` will return all `_deleted: true` jobs. The parameter `size` is the number of documents returned, `from` is how many documents in and `sort` is a lucene query. @@ -444,6 +449,8 @@ $ curl -XPOST 'localhost:5678/v1/jobs/5a50580c-4a50-48d9-80f8-ac70a00f3dbd/_work ## POST /v1/jobs/{jobId}/_active +**DEPRECATED** - Jobs should instead be deleted + Sets the `active` property on the specified job as `true`. **Query Options:** @@ -473,6 +480,8 @@ $ curl -XPOST 'localhost:5678/v1/jobs/5a50580c-4a50-48d9-80f8-ac70a00f3dbd/_acti ## POST /v1/jobs/{jobId}/_inactive +**DEPRECATED** - Jobs should instead be deleted + Sets the `active` property on the specified job as `false`. **Query Options:** @@ -563,6 +572,34 @@ $ curl 'localhost:5678/v1/jobs/5a50580c-4a50-48d9-80f8-ac70a00f3dbd/errors' ] ``` +## DELETE /v1/jobs/\{jobId\}; + +Issues a delete command, deleting the job and all related execution contexts. Deletion is PERMANENT. Once a job is deleted it cannot be started, updated, or recovered. The job must have a terminal status to be deleted. Any orphaned K8s resources associated with the job will also be deleted. The `active` field will automatically be set to `false`. + + +**Usage:** + +```sh +$ curl -XDELETE 'localhost:5678/v1/jobs/5a50580c-4a50-48d9-80f8-ac70a00f3dbd' +{ + "name": "Example", + "lifecycle": "persistent", + "workers": 1, + "operations": [ + { + "_op": "noop" + } + ] + "job_id": "5a50580c-4a50-48d9-80f8-ac70a00f3dbd", + "_context": "job" + "_created": "2018-09-21T17:49:05.029Z", + "_updated": "2019-04-12T09:43:18.301Z", + "_deleted": true, + "_deleted_on": "2019-04-12T09:43:18.301Z", + "active": false, +} +``` + ## GET /v1/ex Returns all execution contexts (job invocations). @@ -573,9 +610,13 @@ Returns all execution contexts (job invocations). - `size: number = 100` - `sort: string = "_updated:desc"` - `status: string = "*"` +- `deleted: string = [true|false]` Size is the number of documents returned, from is how many documents in and sort is a lucene query. +Setting `deleted` to `false` or not setting the option will return execution contexts +where `_deleted` is set to `false` or the `_deleted` key is not present. +Setting `deleted` to `true` will return all execution contexts with `_deleted: true`. **Usage:** ```sh diff --git a/docs/management-apis/endpoints-txt.md b/docs/management-apis/endpoints-txt.md index 7946768f3f7..0d5ae6d9776 100644 --- a/docs/management-apis/endpoints-txt.md +++ b/docs/management-apis/endpoints-txt.md @@ -92,7 +92,10 @@ Returns a text table of all job listings. **Query Options:** - `fields: string` -- `active: [true|false]` +- `active: string = [true|false]` +- `deleted: string = [true|false]` + +**Note:** When showing `deleted` records the `_deleted_on` field will be added to the default fields. The fields parameter is a string that consists of several words, these words will be used to override the default values and only return the values specified ie `fields="job_id,pid"` or `fields="job_id pid"`. @@ -110,6 +113,8 @@ ie `fields="job_id,pid"` or `fields="job_id pid"`. - `job_id` - `_created` - `_updated` +- `_deleted` +- `_deleted_on` **Default Fields:** @@ -139,6 +144,9 @@ Returns a text table of all job execution contexts. **Query Options:** - `fields: string` +- `deleted: string = [true|false]]` + +**Note:** When showing `deleted` records the `_deleted_on` field will be added to the default fields. The fields parameter is a string that consists of several words, these words will be used to override the default values and only return the values specified ie `fields="job_id,pid"` or `fields="job_id pid"`. @@ -156,6 +164,8 @@ ie `fields="job_id,pid"` or `fields="job_id pid"`. - `job_id` - `_created` - `_updated` +- `_deleted` +- `_deleted_on` **Default Fields:** diff --git a/docs/packages/teraslice-cli/overview.md b/docs/packages/teraslice-cli/overview.md index e31fd65c79d..75c331707bf 100644 --- a/docs/packages/teraslice-cli/overview.md +++ b/docs/packages/teraslice-cli/overview.md @@ -294,11 +294,20 @@ teraslice-cli tjm workers remove 5 JOB.JSON teraslice-cli tjm workers total 50 JOB.JSON ``` +### tjm delete + +Delete a job or jobs from a teraslice cluster by referencing the job file. Jobs must be stopped. + +```sh +teraslice-cli tjm delete JOB.JSON +teraslice-cli tjm delete JOB1.JSON JOB2.JSON +``` + ## Jobs *** Job control commands start, stop, pause, resume, and restart all function with the same syntax.*** -- `-all` or `-a` performs action on all the jobs on a given cluster. +- Providing a job_id of `all` will perform the action on all the jobs on a given cluster. - `--yes` or `y` answers yes to all prompts - When jobs are stopped or paused the state of the jobs are saved in `~/.teraslice/job_state_files` @@ -306,7 +315,7 @@ teraslice-cli tjm workers total 50 JOB.JSON Commands: ```bash -teraslice-cli jobs [-all|-a] +teraslice-cli jobs [job_id | all] # stop teraslice-cli jobs stop local 99999999-9999-9999-9999-999999999999 # start @@ -318,7 +327,7 @@ teraslice-cli jobs resume local 99999999-9999-9999-9999-999999999999 # restart job teraslice-cli jobs restart local 99999999-9999-9999-9999-999999999999 # restart all jobs, no prompt -teraslice-cli jobs restart local --all -y +teraslice-cli jobs restart local all -y ``` ### jobs await @@ -372,6 +381,11 @@ Display jobs registered on the cluster teraslice-cli jobs list # list jobs teraslice-cli jobs list local +# list only deleted jobs +teraslice-cli jobs list local --deleted=true +# list only active jobs that have not been deleted +teraslice-cli jobs list local --active=true + ``` ### jobs view @@ -405,6 +419,18 @@ teraslice-cli jobs workers add 5 cluster1 99999999-9999-9999-9999-999999999999 teraslice-cli jobs workers remove 5 cluster1 99999999-9999-9999-9999-999999999999 ``` +### jobs delete + +Delete a job or jobs by job_id from a teraslice cluster. Jobs must be in a terminal state. + +```sh +teraslice-cli jobs delete +# delete a job +teraslice-cli jobs delete cluster1 99999999-9999-9999-9999-999999999999 +# delete all stopped jobs on a cluster, no prompt. Active jobs will be skipped. +teraslice-cli jobs delete cluster1 all -y +``` + ## Executions ### ex errors @@ -436,14 +462,16 @@ teraslice-cli jobs status local --status failing ### ex list -Display execution ids on the cluster, default is `running` and `failing` +Display execution ids on the cluster, default is to exclude deleted and show all statuses ```bash teraslice-cli ex list # list ex_ids teraslice-cli ex list local # list failed ex_ids -teraslice-cli ex list local --status failed +teraslice-cli ex list local --status=failed +# list deleted ex_ids +teraslice-cli ex list local --deleted=true ``` ## Nodes diff --git a/e2e/package.json b/e2e/package.json index eb0fb292eca..514fd07ff37 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -43,9 +43,9 @@ "ms": "^2.1.3" }, "devDependencies": { - "@terascope/types": "^1.0.1", + "@terascope/types": "^1.1.0", "bunyan": "^1.8.15", - "elasticsearch-store": "^1.0.4", + "elasticsearch-store": "^1.1.0", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^3.3.4", diff --git a/e2e/test/cases/cluster/api-spec.ts b/e2e/test/cases/cluster/api-spec.ts index 1f0c7162d4b..44ac87ec95f 100644 --- a/e2e/test/cases/cluster/api-spec.ts +++ b/e2e/test/cases/cluster/api-spec.ts @@ -3,6 +3,7 @@ import { cloneDeep, pDelay } from '@terascope/utils'; import { JobConfig } from '@terascope/types'; import { TerasliceHarness } from '../../teraslice-harness.js'; import { TEST_PLATFORM } from '../../config.js'; +import { Ex, Job } from 'teraslice-client-js'; describe('cluster api', () => { let terasliceHarness: TerasliceHarness; @@ -148,4 +149,71 @@ describe('cluster api', () => { const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1/0.0.1'); expect(response).toBeString(); }); + + describe('DELETE /jobs/', () => { + // NOTE: every test in this section will use a single job + + const deletedJobProperties = { + _deleted: true, + _deleted_on: expect.anything(), + active: false + }; + + let job: Job; + let jobId: string; + let ex: Ex; + let jobSpec: JobConfig; + + beforeAll(async () => { + jobSpec = terasliceHarness.newJob('generator'); + // Set resource constraints on workers within CI + if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') { + jobSpec.resources_requests_cpu = 0.05; + } + + job = await terasliceHarness.teraslice.jobs.submit(jobSpec, false); + jobId = job.id(); + const { ex_id: exId } = await job.execution(); + ex = terasliceHarness.teraslice.executions.wrap(exId); + }) + + it('will not delete a running job', async () => { + await terasliceHarness.waitForExStatus(ex, 'running', 100, 1000); + + await expect(terasliceHarness.teraslice.jobs.delete(`/jobs/${jobId}`)).rejects.toThrow(); + }); + + it('will delete a stopped job', async () => { + await terasliceHarness.teraslice.jobs.post(`/jobs/${jobId}/_stop`); + await terasliceHarness.waitForExStatus(ex, 'stopped', 100, 1000); + + await expect(terasliceHarness.teraslice.jobs.delete(`/jobs/${jobId}`)).resolves.toMatchObject(deletedJobProperties); + }); + + it('will not list a deleted job by default', async () => { + const list = await terasliceHarness.teraslice.jobs.list(); + const jobIds = list.map((job) => job.job_id); + expect(jobIds).toEqual(expect.arrayContaining([expect.not.stringMatching(jobId)])); + }); + + it('will list a deleted job when passed "{ deleted: true }"', async () => { + const list = await terasliceHarness.teraslice.jobs.list({ deleted: true }); + expect(list).toEqual(expect.arrayContaining([expect.objectContaining({ ...jobSpec, job_id: jobId })])); + }); + + it('will not start a deleted job', async () => { + await expect(terasliceHarness.teraslice.jobs.post(`/jobs/${jobId}/_start`)).rejects.toThrow(`Job ${jobId} has been deleted and cannot be started.`); + + }); + + it('will not update a deleted job', async () => { + await expect(terasliceHarness.teraslice.jobs.put(`/jobs/${jobId}`, { workers: 1 })).rejects.toThrow(`Job ${jobId} has been deleted and cannot be updated.`); + + }); + + it('will not recover a deleted job', async () => { + await expect(terasliceHarness.teraslice.jobs.post(`/jobs/${jobId}/_recover`)).rejects.toThrow(`Job ${jobId} has been deleted and cannot be recovered.`); + + }); + }); }); diff --git a/package.json b/package.json index 2736d8e79c4..c4e05652c6a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "2.2.0", + "version": "2.3.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 154b3d791df..201d71f64e8 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "1.0.4", + "version": "1.1.0", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -30,9 +30,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "^1.0.1", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/data-types": "^1.1.0", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "@types/validator": "^13.11.10", "awesome-phonenumber": "^2.70.0", "date-fns": "^2.30.0", @@ -47,7 +47,7 @@ "uuid": "^9.0.1", "valid-url": "^1.0.9", "validator": "^13.12.0", - "xlucene-parser": "^1.0.3" + "xlucene-parser": "^1.1.0" }, "devDependencies": { "@types/ip6addr": "^0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index da6383aa0b1..0f35b972ec8 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "1.0.1", + "version": "1.1.0", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -27,8 +27,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "graphql": "^14.7.0", "lodash": "^4.17.21", "yargs": "^17.7.2" diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index e77c9667ef2..621b9f00982 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "4.0.1", + "version": "4.1.0", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "bluebird": "^3.7.2", "setimmediate": "^1.0.5" }, @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.0.4", + "elasticsearch-store": "^1.1.0", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index 98551bca669..38891740293 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.0.4", + "version": "1.1.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -30,10 +30,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.0.4", - "@terascope/data-types": "^1.0.1", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/data-mate": "^1.1.0", + "@terascope/data-types": "^1.1.0", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "ajv": "^6.12.6", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", @@ -42,7 +42,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1", "setimmediate": "^1.0.5", "uuid": "^9.0.1", - "xlucene-translator": "^1.0.3" + "xlucene-translator": "^1.1.0" }, "devDependencies": { "@types/uuid": "^9.0.8" diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 31b4a153a07..b83e020c967 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "1.2.1", + "version": "1.3.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 3575a33a209..3d02e48a12d 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.0.2", + "version": "1.1.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -33,7 +33,7 @@ }, "dependencies": { "@kubernetes/client-node": "^0.21.0", - "@terascope/utils": "^1.0.1", + "@terascope/utils": "^1.1.0", "codecov": "^3.8.3", "execa": "^5.1.0", "fs-extra": "^11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 9adb21be5e7..ba747361f29 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.2.4", + "version": "1.3.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -29,15 +29,15 @@ }, "dependencies": { "@terascope/file-asset-apis": "^1.0.1", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "bluebird": "^3.7.2", "bunyan": "^1.8.15", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.0.4", + "elasticsearch-store": "^1.1.0", "express": "^4.19.2", "js-yaml": "^4.1.0", "nanoid": "^3.3.4", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 1091dd3d93a..f1bba51a034 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "2.2.1", + "version": "2.3.0", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" @@ -38,8 +38,8 @@ }, "dependencies": { "@terascope/fetch-github-release": "^1.0.0", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "chalk": "^4.1.2", "cli-table3": "^0.6.4", "diff": "^5.2.0", @@ -55,7 +55,7 @@ "pretty-bytes": "^5.6.0", "prompts": "^2.4.2", "signale": "^1.4.0", - "teraslice-client-js": "^1.0.1", + "teraslice-client-js": "^1.1.0", "tmp": "^0.2.0", "tty-table": "^4.2.3", "yargs": "^17.7.2", diff --git a/packages/teraslice-cli/src/cmds/ex/list.ts b/packages/teraslice-cli/src/cmds/ex/list.ts index b5a1cdef2f4..9c28156cba1 100644 --- a/packages/teraslice-cli/src/cmds/ex/list.ts +++ b/packages/teraslice-cli/src/cmds/ex/list.ts @@ -15,9 +15,11 @@ export default { yargs.options('config-dir', yargsOptions.buildOption('config-dir')); yargs.options('output', yargsOptions.buildOption('output')); yargs.options('status', yargsOptions.buildOption('ex-status')); + yargs.options('deleted', yargsOptions.buildOption('show-deleted')); yargs.strict() - .example('$0 ex list cluster1', '') - .example('$0 ex list cluster1 --status=failing', ''); + .example('$0 ex list cluster1', 'Show executions in a cluster') + .example('$0 ex list cluster1 --status=failing', 'Show all executions in cluster with a status of failing') + .example('$0 ex list cluster1 --deleted=true', 'Show only deleted executions in cluster'); return yargs; }, async handler(argv) { @@ -25,22 +27,28 @@ export default { const active = false; const parse = false; const cliConfig = new Config(argv); + const { + clusterAlias, deleted, output, status + } = cliConfig.args; const teraslice = new TerasliceUtil(cliConfig); - const format = `${cliConfig.args.output}Horizontal`; + const format = `${output}Horizontal`; const header = ['name', 'lifecycle', 'slicers', 'workers', '_status', 'ex_id', 'job_id', '_created', '_updated']; + if (deleted === true) { + header.push('_deleted_on'); + } try { - response = await teraslice.client.executions.list(cliConfig.args.status); + response = await teraslice.client.executions.list({ status, deleted }); } catch (err) { - reply.fatal(`Error getting ex list on ${cliConfig.args.clusterAlias}\n${err}`); + reply.fatal(`Error getting ex list on ${clusterAlias}\n${err}`); } const rows = await display.parseResponse(header, response ?? [], active); if (rows.length > 0) { await display.display(header, rows, format, active, parse); } else { - reply.fatal(`> No ex_ids match status ${cliConfig.args.status}`); + reply.fatal(`> No ex_ids match "status: ${status !== '' ? status : '*'}" and "deleted: ${deleted}"`); } } } as CMD; diff --git a/packages/teraslice-cli/src/cmds/jobs/delete.ts b/packages/teraslice-cli/src/cmds/jobs/delete.ts new file mode 100644 index 00000000000..666a971f970 --- /dev/null +++ b/packages/teraslice-cli/src/cmds/jobs/delete.ts @@ -0,0 +1,36 @@ +import { CMD } from '../../interfaces.js'; +import Config from '../../helpers/config.js'; +import YargsOptions from '../../helpers/yargs-options.js'; +import Jobs from '../../helpers/jobs.js'; +import reply from '../../helpers/reply.js'; + +const yargsOptions = new YargsOptions(); + +export default { + command: 'delete ', + describe: 'Delete jobs from the teraslice cluster. Jobs must be in a terminal state.', + builder(yargs: any) { + yargs.positional('job-id', yargsOptions.buildPositional('job-id')); + yargs.options('config-dir', yargsOptions.buildOption('config-dir')); + yargs.options('output', yargsOptions.buildOption('output')); + yargs.options('yes', yargsOptions.buildOption('yes')); + yargs.strict() + .example('$0 jobs delete CLUSTER_ALIAS JOB_ID', 'deletes job on a cluster') + .example('$0 jobs delete CLUSTER_ALIAS JOB_ID1 JOB_ID2', 'deletes multiple jobs on a cluster') + .example('$0 jobs delete CLUSTER_ALIAS all', 'deletes all non-active jobs on a cluster'); + return yargs; + }, + async handler(argv: any) { + const cliConfig = new Config(argv); + + const jobs = new Jobs(cliConfig); + + await jobs.initialize(); + + try { + await jobs.delete(); + } catch (e) { + reply.fatal(e); + } + } +} as CMD; diff --git a/packages/teraslice-cli/src/cmds/jobs/index.ts b/packages/teraslice-cli/src/cmds/jobs/index.ts index 3f0f093beee..2e082f62c12 100644 --- a/packages/teraslice-cli/src/cmds/jobs/index.ts +++ b/packages/teraslice-cli/src/cmds/jobs/index.ts @@ -1,5 +1,6 @@ import { CMD } from '../../interfaces.js'; import awaitCmd from './await.js'; +import deleteJob from './delete.js'; import errors from './errors.js'; import list from './list.js'; import pause from './pause.js'; @@ -16,6 +17,7 @@ import workers from './workers.js'; const commandList = [ awaitCmd, + deleteJob, errors, list, pause, diff --git a/packages/teraslice-cli/src/cmds/jobs/list.ts b/packages/teraslice-cli/src/cmds/jobs/list.ts index 74ddd6587e6..631f074bfa4 100644 --- a/packages/teraslice-cli/src/cmds/jobs/list.ts +++ b/packages/teraslice-cli/src/cmds/jobs/list.ts @@ -15,29 +15,39 @@ export default { builder(yargs: any) { yargs.options('config-dir', yargsOptions.buildOption('config-dir')); yargs.options('output', yargsOptions.buildOption('output')); + yargs.options('deleted', yargsOptions.buildOption('show-deleted')); + yargs.options('active', yargsOptions.buildOption('active-job')); yargs.strict() - .example('$0 jobs list CLUSTER_ALIAS'); + .example('$0 jobs list CLUSTER_ALIAS') + .example('$0 jobs list CLUSTER_ALIAS --deleted=true', 'Show only deleted jobs in cluster') + .example('$0 jobs list CLUSTER_ALIAS --active=true', 'Show only active jobs in cluster'); return yargs; }, async handler(argv: any) { let response: Teraslice.JobConfigParams[]; - const active = false; + const displayActive = false; const parse = true; const cliConfig = new Config(argv); + const { + active, clusterAlias, deleted, output + } = cliConfig.args; const teraslice = new TerasliceUtil(cliConfig); const header = ['job_id', 'name', 'lifecycle', 'slicers', 'workers', '_created', '_updated']; - const format = `${cliConfig.args.output}Horizontal`; + const format = `${output}Horizontal`; + if (deleted === true) { + header.push('_deleted_on'); + } try { - response = await teraslice.client.jobs.list(); + response = await teraslice.client.jobs.list({ active, deleted }); } catch (err) { - reply.fatal(`Error getting jobs list on ${cliConfig.args.clusterAlias}\n${err}`); + reply.fatal(`Error getting jobs list on ${clusterAlias}\n${err}`); } // @ts-expect-error if (response.length === 0) { - reply.fatal(`> No jobs on ${cliConfig.args.clusterAlias}`); + reply.fatal(`> No jobs on ${clusterAlias} match with "deleted: ${deleted}" and "active: ${active}"`); } // @ts-expect-error - await display.display(header, response, format, active, parse); + await display.display(header, response, format, displayActive, parse); } } as CMD; diff --git a/packages/teraslice-cli/src/cmds/tjm/delete.ts b/packages/teraslice-cli/src/cmds/tjm/delete.ts new file mode 100644 index 00000000000..6c9e5837601 --- /dev/null +++ b/packages/teraslice-cli/src/cmds/tjm/delete.ts @@ -0,0 +1,33 @@ +import { CMD } from '../../interfaces.js'; +import Config from '../../helpers/config.js'; +import YargsOptions from '../../helpers/yargs-options.js'; +import { validateAndUpdateCliConfig } from '../../helpers/tjm-util.js'; +import Jobs from '../../helpers/jobs.js'; + +const yargsOptions = new YargsOptions(); + +export default { + command: 'delete ', + describe: 'Delete a job or jobs by referencing the job file. Jobs must be in a terminal state.', + builder(yargs: any) { + yargs.positional('job-file', yargsOptions.buildPositional('job-file')); + yargs.option('src-dir', yargsOptions.buildOption('src-dir')); + yargs.options('config-dir', yargsOptions.buildOption('config-dir')); + yargs.options('yes', yargsOptions.buildOption('yes')); + yargs + .example('$0 tjm delete JOB_FILE.json', 'deletes a job') + .example('$0 tjm delete JOB_FILE.json JOB_FILE2.json', 'deletes multiple jobs'); + return yargs; + }, + async handler(argv: any) { + const cliConfig = new Config(argv); + + validateAndUpdateCliConfig(cliConfig); + + const jobs = new Jobs(cliConfig); + + await jobs.initialize(); + + await jobs.delete(); + } +} as CMD; diff --git a/packages/teraslice-cli/src/cmds/tjm/index.ts b/packages/teraslice-cli/src/cmds/tjm/index.ts index 4cc8e37f5e3..1476f9cc4e9 100644 --- a/packages/teraslice-cli/src/cmds/tjm/index.ts +++ b/packages/teraslice-cli/src/cmds/tjm/index.ts @@ -1,6 +1,7 @@ import { CMD } from '../../interfaces.js'; import awaitCmd from './await.js'; import convert from './convert.js'; +import deleteJob from './delete.js'; import errors from './errors.js'; import init from './init.js'; import pause from './pause.js'; @@ -18,6 +19,7 @@ import workers from './workers.js'; const commandList = [ awaitCmd, convert, + deleteJob, errors, init, pause, diff --git a/packages/teraslice-cli/src/helpers/jobs.ts b/packages/teraslice-cli/src/helpers/jobs.ts index 63f498ffcd8..f642127e12f 100644 --- a/packages/teraslice-cli/src/helpers/jobs.ts +++ b/packages/teraslice-cli/src/helpers/jobs.ts @@ -493,6 +493,42 @@ export default class Jobs { } } + async delete(): Promise { + if (!this.config.args.jobId.includes('all') && this.config.args.yes !== true) { + const jobsString = this.jobs.length === 1 + ? `job ${this.jobs[0].id}` + : `jobs ${this.jobs.map((job) => job.id).join(', ')}`; + const prompt = await display.showPrompt( + this.config.args._action, + `${jobsString} on ${this.config.clusterUrl}` + ); + if (!prompt) return; + } + + await pMap( + this.jobs, + (job) => this.deleteOne(job), + { concurrency: this.concurrency } + ); + } + + async deleteOne(job: JobMetadata) { + if (!this.inTerminalStatus(job)) { + const { jobInfoString } = this.getJobIdentifiers(job); + + reply.error(`Job is in non-terminal status ${job.status}, cannot delete. Skipping\n${jobInfoString}`); + return; + } + + try { + await job.api.deleteJob(); + } catch (e) { + this.commandFailed(e.message, job); + } + + this.logUpdate({ action: 'deleted', job }); + } + private inTerminalStatus(job: JobMetadata): boolean { return this.terminalStatuses.includes(job.status); } @@ -551,16 +587,23 @@ export default class Jobs { private async getAllJobs() { if (await this.prompt()) { + const { _action: action, clusterAlias } = this.config.args; // if action is start and not from a restart // then need to get job ids from saved state - if (this.config.args._action === 'start') { + if (action === 'start') { if (fs.pathExistsSync(this.config.jobStateFile) === false) { - reply.fatal(`Could not find job state file for ${this.config.args.clusterAlias}, this is required to start all jobs`); + reply.fatal(`Could not find job state file for ${clusterAlias}, this is required to ${action} all jobs`); } return this.getJobIdsFromSavedState(); } + // if action is delete we need to get inactive + // as well as active jobs + if (action === 'delete') { + return this.getActiveAndInactiveJobIds(); + } + return this.getActiveJobIds(); } @@ -574,6 +617,15 @@ export default class Jobs { return Object.keys(state); } + private async getActiveAndInactiveJobIds() { + try { + const jobs = await this.teraslice.client.jobs.list(); + return jobs.map((job) => job.job_id); + } catch (e) { + throw Error(e); + } + } + private async getActiveJobIds(): Promise { const controllers = await this.getClusterControllers(); @@ -859,6 +911,10 @@ export default class Jobs { cannot_stop: { message: `No need to stop, job is already in terminal status ${status}`, final: this.finalAction(action) + }, + deleted: { + message: `${name} has been deleted`, + final: true } }; diff --git a/packages/teraslice-cli/src/helpers/yargs-options.ts b/packages/teraslice-cli/src/helpers/yargs-options.ts index ec4e451f3da..c044b76fd1d 100644 --- a/packages/teraslice-cli/src/helpers/yargs-options.ts +++ b/packages/teraslice-cli/src/helpers/yargs-options.ts @@ -232,6 +232,16 @@ export default class Options { describe: 'Shows diff on a job in a cluster and a job file', default: false, type: 'boolean' + }), + 'active-job': () => ({ + describe: 'List active jobs', + type: 'boolean' + + }), + 'show-deleted': () => ({ + describe: 'List deleted records', + default: false, + type: 'boolean' }) }; diff --git a/packages/teraslice-cli/test/cmds/jobs/delete-spec.ts b/packages/teraslice-cli/test/cmds/jobs/delete-spec.ts new file mode 100644 index 00000000000..d6624c52fe4 --- /dev/null +++ b/packages/teraslice-cli/test/cmds/jobs/delete-spec.ts @@ -0,0 +1,34 @@ +import yargs from 'yargs'; +import deleteJob from '../../../src/cmds/jobs/delete.js'; + +describe('jobs delete', () => { + describe('-> parse', () => { + it('should parse properly', () => { + const yargsCmd = yargs().command( + // @ts-expect-error + deleteJob.command, + deleteJob.describe, + deleteJob.builder, + () => true + ); + const yargsResult = yargsCmd.parseSync( + 'delete ts-test1 all', {} + ); + expect(yargsResult.clusterAlias).toEqual('ts-test1'); + }); + it('should parse properly with an id specifed', () => { + const yargsCmd = yargs().command( + // @ts-expect-error + deleteJob.command, + deleteJob.describe, + deleteJob.builder, + () => true + ); + const yargsResult = yargsCmd.parseSync( + 'delete ts-test1 99999999-9999-9999-9999-999999999999', {} + ); + expect(yargsResult.clusterAlias).toEqual('ts-test1'); + expect(yargsResult.jobId).toEqual(['99999999-9999-9999-9999-999999999999']); + }); + }); +}); diff --git a/packages/teraslice-cli/test/helpers/jobs-spec.ts b/packages/teraslice-cli/test/helpers/jobs-spec.ts index 3e30f472347..ca243487783 100644 --- a/packages/teraslice-cli/test/helpers/jobs-spec.ts +++ b/packages/teraslice-cli/test/helpers/jobs-spec.ts @@ -1,3 +1,4 @@ +import { jest } from '@jest/globals'; import nock from 'nock'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -10,6 +11,7 @@ import { clusterControllers, getJobExecution } from './helpers.js'; +import reply from '../../src/helpers/reply.js'; const dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -879,4 +881,67 @@ describe('Job helper class', () => { expect(jobs).toBeDefined(); }); }); + + describe('delete', () => { + const action = 'delete'; + it('should delete a stopped job', async () => { + const [jobId] = makeJobIds(1); + + tsClient + .get(`/v1/jobs/${jobId}/ex`) + .reply(200, { _status: 'stopped' }) + .get(`/v1/jobs/${jobId}`) + .reply(200, testJobConfig(jobId)) + .delete(`/v1/jobs/${jobId}`) + .reply(200, () => Promise.resolve({ _deleted: 'true' })); + + const config = buildCLIConfig( + action, + { + 'job-id': [jobId], + jobId: [jobId], + yes: true, + y: true + } + ); + + const job = new Jobs(config); + + await job.initialize(); + + expect(job.jobs[0].status).toBe('stopped'); + + await expect(job.delete()).resolves.toBeUndefined(); + }); + + it('should log an error if job is not in a terminal status', async () => { + reply.error = jest.fn() + const [jobId] = makeJobIds(1); + + tsClient + .get(`/v1/jobs/${jobId}/ex`) + .reply(200, { _status: 'running' }) + .get(`/v1/jobs/${jobId}`) + .reply(200, testJobConfig(jobId)); + + const config = buildCLIConfig( + action, + { + 'job-id': [jobId], + jobId: [jobId], + yes: true, + y: true + } + ); + + const job = new Jobs(config); + + await job.initialize(); + + expect(job.jobs[0].status).toBe('running'); + + await expect(job.delete()).resolves.toBe(undefined); + expect(reply.error).toHaveBeenCalledWith(expect.stringContaining('Job is in non-terminal status running, cannot delete. Skipping')) + }); + }); }); diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index 5ccdf05af0f..52bc5e8ca7c 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "1.0.1", + "version": "1.1.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "auto-bind": "^4.0.0", "got": "^11.8.3" }, diff --git a/packages/teraslice-client-js/src/job.ts b/packages/teraslice-client-js/src/job.ts index f7c343cc771..4b72513827f 100644 --- a/packages/teraslice-client-js/src/job.ts +++ b/packages/teraslice-client-js/src/job.ts @@ -101,6 +101,10 @@ export default class Job extends Client { return this.update(body); } + async deleteJob(): Promise { + return this.delete(`/jobs/${this._jobId}`); + } + async execution(requestOptions: RequestOptions = {}): Promise { return this.get(`/jobs/${this._jobId}/ex`, requestOptions); } diff --git a/packages/teraslice-client-js/src/jobs.ts b/packages/teraslice-client-js/src/jobs.ts index b4629569da6..8d656674e69 100644 --- a/packages/teraslice-client-js/src/jobs.ts +++ b/packages/teraslice-client-js/src/jobs.ts @@ -1,4 +1,4 @@ -import { isString, TSError } from '@terascope/utils'; +import { TSError } from '@terascope/utils'; import { Teraslice } from '@terascope/types'; import autoBind from 'auto-bind'; import Client from './client.js'; @@ -26,10 +26,9 @@ export default class Jobs extends Client { } async list( - status?: Teraslice.JobListStatusQuery, + query?: Teraslice.JobSearchParams, searchOptions: SearchOptions = {} ): Promise { - const query = _parseListOptions(status); return this.get('/jobs', this.makeOptions(query, searchOptions)); } @@ -41,10 +40,3 @@ export default class Jobs extends Client { return new Job(this._config, jobId); } } - -function _parseListOptions(options?: Teraslice.JobListStatusQuery): Teraslice.JobSearchParams { - // support legacy - if (!options) return { status: '*' }; - if (isString(options)) return { status: options }; - return options; -} diff --git a/packages/teraslice-client-js/test/job-spec.ts b/packages/teraslice-client-js/test/job-spec.ts index 59e693cc30b..deb9ec5a962 100644 --- a/packages/teraslice-client-js/test/job-spec.ts +++ b/packages/teraslice-client-js/test/job-spec.ts @@ -325,7 +325,7 @@ describe('Teraslice Job', () => { _context: 'job', job_id: 'some-job-id', _created: 'hello', - _updated: 'hello', + _updated: 'hello' }; beforeEach(() => { @@ -360,7 +360,7 @@ describe('Teraslice Job', () => { _context: 'job', job_id: 'some-job-id', _created: 'hello', - _updated: 'hello', + _updated: 'hello' }; const expected = { @@ -383,6 +383,21 @@ describe('Teraslice Job', () => { }); }); + describe('->deleteJob', () => { + describe('when called', () => { + beforeEach(() => { + scope.delete('/jobs/some-other-job-id') + .reply(200, { job_id: 'some-other-job-id', _deleted: true }); + }); + + it('should resolve json results from Teraslice', async () => { + const job = new Job({ baseUrl }, 'some-other-job-id'); + const results = await job.deleteJob(); + expect(results).toEqual({ job_id: 'some-other-job-id', _deleted: true }); + }); + }); + }); + describe('->exId', () => { describe('when called with nothing', () => { beforeEach(() => { diff --git a/packages/teraslice-client-js/test/jobs-spec.ts b/packages/teraslice-client-js/test/jobs-spec.ts index 2c585418701..af80ea2a985 100644 --- a/packages/teraslice-client-js/test/jobs-spec.ts +++ b/packages/teraslice-client-js/test/jobs-spec.ts @@ -170,7 +170,6 @@ describe('Teraslice Jobs', () => { describe('when called with nothing', () => { beforeEach(() => { scope.get('/jobs') - .query({ status: '*' }) .reply(200, list); }); @@ -180,22 +179,9 @@ describe('Teraslice Jobs', () => { }); }); - describe('when called with a string', () => { - beforeEach(() => { - scope.get('/jobs') - .query({ status: Teraslice.ExecutionStatusEnum.running }) - .reply(200, list); - }); - - it('should resolve json result from Teraslice', async () => { - const results = await jobs.list(Teraslice.ExecutionStatusEnum.running); - expect(results).toEqual(list); - }); - }); - - describe('when called with an object', () => { + describe('when called with an query and search objects', () => { const searchOptions = { headers: { 'Some-Header': 'yes' } }; - const queryOptions = { status: Teraslice.ExecutionStatusEnum.running, size: 10 }; + const queryOptions = { active: true } as const; beforeEach(() => { scope.get('/jobs') diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index 5827ba1c8db..8c65ed01797 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "1.3.1", + "version": "1.4.0", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -35,8 +35,8 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "ms": "^2.1.3", "nanoid": "^3.3.4", "p-event": "^4.2.0", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index 5f51dbffc84..ca07eca4807 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "1.0.1", + "version": "1.1.0", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "^4.0.1", - "@terascope/utils": "^1.0.1" + "@terascope/elasticsearch-api": "^4.1.0", + "@terascope/utils": "^1.1.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index b1b58fd3e59..493e4060bce 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "^11.2.0" }, "devDependencies": { - "@terascope/job-components": "^1.2.1" + "@terascope/job-components": "^1.3.0" }, "peerDependencies": { - "@terascope/job-components": ">=1.2.1" + "@terascope/job-components": ">=1.3.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 85d254f3bb3..f13c48d027e 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "2.2.0", + "version": "2.3.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { @@ -39,11 +39,11 @@ }, "dependencies": { "@kubernetes/client-node": "^0.21.0", - "@terascope/elasticsearch-api": "^4.0.1", - "@terascope/job-components": "^1.2.1", - "@terascope/teraslice-messaging": "^1.3.1", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/elasticsearch-api": "^4.1.0", + "@terascope/job-components": "^1.3.0", + "@terascope/teraslice-messaging": "^1.4.0", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "async-mutex": "^0.5.0", "barbe": "^3.0.16", "body-parser": "^1.20.2", @@ -64,7 +64,7 @@ "semver": "^7.6.3", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^1.2.4", + "terafoundation": "^1.3.0", "uuid": "^9.0.1" }, "devDependencies": { diff --git a/packages/teraslice/src/lib/cluster/services/api.ts b/packages/teraslice/src/lib/cluster/services/api.ts index 75e2a90309b..13c335e8592 100644 --- a/packages/teraslice/src/lib/cluster/services/api.ts +++ b/packages/teraslice/src/lib/cluster/services/api.ts @@ -14,6 +14,7 @@ import type { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/ import { makePrometheus, isPrometheusTerasliceRequest, makeTable, sendError, handleTerasliceRequest, getSearchOptions, + createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js'; import { getPackageJSON } from '../../utils/file_utils.js'; @@ -40,6 +41,14 @@ function validateCleanupType(cleanupType: RecoveryCleanupType) { } } +function validateGetDeletedOption(deletedOption: string) { + if (!['true', 'false', ''].includes(deletedOption)) { + throw new TSError('deleted query option must true or false', { + statusCode: 400 + }); + } +} + export class ApiService { context: ClusterMasterContext; logger: Logger; @@ -275,19 +284,18 @@ export class ApiService { }); v1routes.get('/jobs', (req, res) => { - let query: string; + const { active = '', deleted = 'false' } = req.query; const { size, from, sort } = getSearchOptions(req as TerasliceRequest); - if (req.query.active === 'true') { - query = 'job_id:* AND !active:false'; - } else if (req.query.active === 'false') { - query = 'job_id:* AND active:false'; - } else { - query = 'job_id:*'; - } - const requestHandler = handleTerasliceRequest(req as TerasliceRequest, res, 'Could not retrieve list of jobs'); - requestHandler(() => this.jobsStorage.search(query, from, size, sort as string)); + requestHandler(() => { + validateGetDeletedOption(deleted as string); + + const partialQuery = createJobActiveQuery(active as string); + const query = addDeletedToQuery(deleted as string, partialQuery); + + return this.jobsStorage.search(query, from, size, sort as string); + }); }); v1routes.get('/jobs/:jobId', (req, res) => { @@ -379,6 +387,13 @@ export class ApiService { }); }); + v1routes.delete('/jobs/:jobId', (req, res) => { + const { jobId } = req.params; + // @ts-expect-error + const requestHandler = handleTerasliceRequest(req as TerasliceRequest, res, 'Could not delete job'); + requestHandler(async () => jobsService.softDeleteJob(jobId)); + }); + v1routes.post('/jobs/:jobId/_recover', (req, res) => { const cleanupType = req.query.cleanup_type || req.query.cleanup; const { jobId } = req.params; @@ -443,20 +458,23 @@ export class ApiService { }); v1routes.get('/ex', (req, res) => { - const { status = '' } = req.query; + const { status = '', deleted = 'false' } = req.query; const { size, from, sort } = getSearchOptions(req as TerasliceRequest); const requestHandler = handleTerasliceRequest(req as TerasliceRequest, res, 'Could not retrieve list of execution contexts'); requestHandler(async () => { + validateGetDeletedOption(deleted as string); const statuses = parseList(status); - let query = 'ex_id:*'; + let partialQuery = 'ex_id:*'; if (statuses.length) { const statusTerms = statuses.map((s) => `_status:"${s}"`).join(' OR '); - query += ` AND (${statusTerms})`; + partialQuery += ` AND (${statusTerms})`; } + const query = addDeletedToQuery(deleted as string, partialQuery); + return this.executionStorage.search(query, from, size, sort as string); }); }); @@ -536,21 +554,22 @@ export class ApiService { }); this.app.get('/txt/jobs', (req, res) => { - let query: string; + const { active = '', deleted = 'false' } = req.query; const { size, from, sort } = getSearchOptions(req as TerasliceRequest); const defaults = ['job_id', 'name', 'active', 'lifecycle', 'slicers', 'workers', '_created', '_updated']; - if (req.query.active === 'true') { - query = 'job_id:* AND !active:false'; - } else if (req.query.active === 'false') { - query = 'job_id:* AND active:false'; - } else { - query = 'job_id:*'; - } - const requestHandler = handleTerasliceRequest(req as TerasliceRequest, res, 'Could not get all jobs'); requestHandler(async () => { + validateGetDeletedOption(deleted as string); + + if (deleted !== 'false') { + defaults.push('_deleted_on'); + } + + const partialQuery = createJobActiveQuery(active as string); + const query = addDeletedToQuery(deleted as string, partialQuery); + const jobs = await this.jobsStorage.search( query, from, size, sort as string ) as Record[]; @@ -560,14 +579,23 @@ export class ApiService { }); this.app.get('/txt/ex', (req, res) => { + const { deleted = 'false' } = req.query; const { size, from, sort } = getSearchOptions(req as TerasliceRequest); const defaults = ['name', 'lifecycle', 'slicers', 'workers', '_status', 'ex_id', 'job_id', '_created', '_updated']; - const query = 'ex_id:*'; const requestHandler = handleTerasliceRequest(req as TerasliceRequest, res, 'Could not get all executions'); requestHandler(async () => { + validateGetDeletedOption(deleted as string); + + if (deleted !== 'false') { + defaults.push('_deleted_on'); + } + + const partialQuery = 'ex_id:*'; + const query = addDeletedToQuery(deleted as string, partialQuery); + const exs = await this.executionStorage.search( query, from, size, sort as string ) as Record[]; diff --git a/packages/teraslice/src/lib/cluster/services/execution.ts b/packages/teraslice/src/lib/cluster/services/execution.ts index 1292a97bec5..1db19012927 100644 --- a/packages/teraslice/src/lib/cluster/services/execution.ts +++ b/packages/teraslice/src/lib/cluster/services/execution.ts @@ -395,6 +395,16 @@ export class ExecutionService { } } + async softDeleteExecutionContext(exId: string): Promise { + const exIds = await this.getRunningExecutions(exId); + if (exIds.length > 0) { + throw new TSError(`Execution ${exId} is currently running, cannot delete a running execution.`, { + statusCode: 409 + }); + } + return this.executionStorage.softDelete(exId); + } + async getRunningExecutions(exId: string | undefined) { let query = this.executionStorage.getRunningStatuses().map((state) => ` _status:${state} `).join('OR'); diff --git a/packages/teraslice/src/lib/cluster/services/jobs.ts b/packages/teraslice/src/lib/cluster/services/jobs.ts index 619fb7e0cc2..637560fa53d 100644 --- a/packages/teraslice/src/lib/cluster/services/jobs.ts +++ b/packages/teraslice/src/lib/cluster/services/jobs.ts @@ -2,7 +2,7 @@ import defaultsDeep from 'lodash/defaultsDeep.js'; import { TSError, uniq, cloneDeep, isEmpty, getTypeOf, isString, - Logger + Logger, makeISODate } from '@terascope/utils'; import { JobConfigParams, JobValidator, RecoveryCleanupType, @@ -114,10 +114,13 @@ export class JobsService { } async updateJob(jobId: string, jobSpec: Partial) { - await this._validateJobSpec(jobSpec); - const originalJob = await this.jobsStorage.get(jobId); + if (originalJob._deleted === true) { + throw new TSError(`Job ${jobId} has been deleted and cannot be updated.`, { + statusCode: 410 + }); + } // If job is switching from active to inactive job validation is skipped // This allows for old jobs that are missing required resources to be marked inactive if (originalJob.active !== false && jobSpec.active === false) { @@ -164,6 +167,13 @@ export class JobsService { } const jobSpec = await this.jobsStorage.get(jobId); + + if (jobSpec._deleted === true) { + throw new TSError(`Job ${jobId} has been deleted and cannot be started.`, { + statusCode: 410 + }); + } + const validJob = await this._validateJobSpec(jobSpec) as JobConfig; if (validJob.autorecover) { @@ -218,6 +228,13 @@ export class JobsService { async recoverJob(jobId: string, cleanupType: RecoveryCleanupType) { // we need to do validations since the job config could change between recovery const jobSpec = await this.jobsStorage.get(jobId); + + if (jobSpec._deleted === true) { + throw new TSError(`Job ${jobId} has been deleted and cannot be recovered.`, { + statusCode: 410 + }); + } + const validJob = await this._validateJobSpec(jobSpec) as JobConfig; return this._recoverValidJob(validJob, cleanupType); @@ -233,13 +250,91 @@ export class JobsService { return this.executionService.resumeExecution(exId); } + async softDeleteJob(jobId: string) { + const activeExecution = await this._getActiveExecution(jobId, true); + + // searching for an active execution, if there is then we reject + if (activeExecution) { + throw new TSError(`Job ${jobId} is currently running, cannot delete a running job.`, { + statusCode: 409 + }); + } + + // This will return any orphaned resources in k8s clustering + // or an empty array in native clustering + let currentResources = await this.executionService.listResourcesForJobId(jobId); + + if (currentResources.length > 0) { + currentResources = currentResources.flat(); + const exIdsSet = new Set(); + for (const resource of currentResources) { + exIdsSet.add(resource.metadata.labels['teraslice.terascope.io/exId']); + } + const exIdsArr = Array.from(exIdsSet); + const exIdsString = exIdsArr.join(', '); + this.logger.info(`There are orphaned resources for job: ${jobId}, exId: ${exIdsString}.\n` + + 'Removing resources before job deletion.'); + await Promise.all(exIdsArr + .map((exId) => this.executionService.stopExecution(exId, { force: true })) + ); + } + + const jobSpec = await this.jobsStorage.get(jobId); + + if (jobSpec._deleted === true) { + throw new TSError(`Job ${jobId} has already been deleted.`, { + statusCode: 410 + }); + } + + jobSpec._deleted = true; + jobSpec._deleted_on = makeISODate(); + jobSpec.active = false; + + const executions = await this.getAllExecutions(jobId, undefined, true); + for (const execution of executions) { + await this.executionService.softDeleteExecutionContext(execution.ex_id); + } + return this.jobsStorage.update(jobId, jobSpec); + } + + /** + * Get all executions related to a jobId + * + * @param {string} jobId + * @param {string} [query] + * @param {boolean=false} [allowZeroResults] + * @returns {Promise} + */ + async getAllExecutions( + jobId: string, + query?: string, + allowZeroResults = false + ): Promise { + if (!jobId || !isString(jobId)) { + throw new TSError(`Invalid job id, got ${getTypeOf(jobId)}`); + } + + const executions = await this.executionStorage.search( + query || `job_id: "${jobId}"`, undefined, undefined, '_created:desc' + ) as ExecutionConfig[]; + + if (!allowZeroResults && !executions.length) { + throw new TSError(`No executions were found for job ${jobId}`, { + statusCode: 404 + }); + } + + return executions; + } + /** * Get the latest execution * * @param {string} jobId * @param {string} [query] * @param {boolean=false} [allowZeroResults] - * @returns {Promise} + * @returns {Promise} */ async getLatestExecution( jobId: string, @@ -268,7 +363,7 @@ export class JobsService { * * @param {string} jobId * @param {boolean} [allowZeroResults] - * @returns {Promise} + * @returns {Promise} */ private async _getActiveExecution(jobId: string, allowZeroResults?: boolean) { const statuses = this.executionStorage @@ -285,8 +380,7 @@ export class JobsService { * Get the active execution * * @param {string} jobId - * @param {boolean} [allowZeroResults] - * @returns {Promise} + * @returns {Promise} */ private async _getActiveExecutionId(jobId: string): Promise { const { ex_id } = await this._getActiveExecution(jobId); diff --git a/packages/teraslice/src/lib/storage/backends/mappings/ex.ts b/packages/teraslice/src/lib/storage/backends/mappings/ex.ts index 07b03befb31..5b0e9516815 100644 --- a/packages/teraslice/src/lib/storage/backends/mappings/ex.ts +++ b/packages/teraslice/src/lib/storage/backends/mappings/ex.ts @@ -48,6 +48,12 @@ export default { }, _updated: { type: 'date' + }, + _deleted: { + type: 'boolean' + }, + _deleted_on: { + type: 'date' } } } diff --git a/packages/teraslice/src/lib/storage/backends/mappings/job.ts b/packages/teraslice/src/lib/storage/backends/mappings/job.ts index 296affe0095..be910fe0be6 100644 --- a/packages/teraslice/src/lib/storage/backends/mappings/job.ts +++ b/packages/teraslice/src/lib/storage/backends/mappings/job.ts @@ -24,6 +24,12 @@ export default { }, _updated: { type: 'date' + }, + _deleted: { + type: 'boolean' + }, + _deleted_on: { + type: 'date' } } } diff --git a/packages/teraslice/src/lib/storage/execution.ts b/packages/teraslice/src/lib/storage/execution.ts index 143a15f16d4..afbc4156283 100644 --- a/packages/teraslice/src/lib/storage/execution.ts +++ b/packages/teraslice/src/lib/storage/execution.ts @@ -262,6 +262,25 @@ export class ExecutionStorage { } } + async softDelete(exId: string) { + try { + const date = makeISODate(); + return await this.updatePartial( + exId, + async (existing) => Object.assign(existing, { + _deleted: true, + _deleted_on: date, + _updated: date + }) + ); + } catch (err) { + throw new TSError(err, { + statusCode: 422, + reason: `Unable to delete execution ${exId}` + }); + } + } + async remove(exId: string) { return this.backend.remove(exId); } diff --git a/packages/teraslice/src/lib/utils/api_utils.ts b/packages/teraslice/src/lib/utils/api_utils.ts index ebba35e2f6c..c7f4adc7233 100644 --- a/packages/teraslice/src/lib/utils/api_utils.ts +++ b/packages/teraslice/src/lib/utils/api_utils.ts @@ -170,3 +170,20 @@ export function logTerasliceRequest(req: TerasliceRequest) { const { method, path } = req; req.logger.trace(`${method.toUpperCase()} ${path} endpoint has been called, ${queryInfo}`); } + +export function createJobActiveQuery(active: string) { + if (active === 'true') { + return 'job_id:* AND !active:false'; + } + if (active === 'false') { + return 'job_id:* AND active:false'; + } + return 'job_id:*'; +} + +export function addDeletedToQuery(deleted: string, query: string) { + if (deleted === 'false') { + return `${query} AND (_deleted:false OR (* AND -_deleted:*))`; + } + return `${query} AND _deleted:true`; +} diff --git a/packages/teraslice/test/utils/api_utils-spec.ts b/packages/teraslice/test/utils/api_utils-spec.ts index 4680c028f91..ace60fe915a 100644 --- a/packages/teraslice/test/utils/api_utils-spec.ts +++ b/packages/teraslice/test/utils/api_utils-spec.ts @@ -1,4 +1,6 @@ -import { makePrometheus, isPrometheusTerasliceRequest } from '../../src/lib/utils/api_utils.js'; +import { + makePrometheus, isPrometheusTerasliceRequest, createJobActiveQuery, addDeletedToQuery +} from '../../src/lib/utils/api_utils.js'; describe('apiUtils', () => { it('should be able make a prometheus text format', () => { @@ -64,4 +66,32 @@ teraslice_workers_reconnected{foo="bar"} ${stats.controllers.workers_reconnected } } as any)).toBeTruthy(); }); + + it('should be able to create the proper job queries', () => { + let query: string; + + query = createJobActiveQuery('true'); + query = addDeletedToQuery('true', query); + expect(query).toBe('job_id:* AND !active:false AND _deleted:true'); + + query = createJobActiveQuery('true'); + query = addDeletedToQuery('', query); + expect(query).toBe('job_id:* AND !active:false AND _deleted:true'); + + query = createJobActiveQuery('true'); + query = addDeletedToQuery('false', query); + expect(query).toBe('job_id:* AND !active:false AND (_deleted:false OR (* AND -_deleted:*))'); + + query = createJobActiveQuery('false'); + query = addDeletedToQuery('true', query); + expect(query).toBe('job_id:* AND active:false AND _deleted:true'); + + query = createJobActiveQuery('false'); + query = addDeletedToQuery('', query); + expect(query).toBe('job_id:* AND active:false AND _deleted:true'); + + query = createJobActiveQuery('false'); + query = addDeletedToQuery('false', query); + expect(query).toBe('job_id:* AND active:false AND (_deleted:false OR (* AND -_deleted:*))'); + }); }); diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index dd0e2d10166..c8943d03f3b 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "1.0.4", + "version": "1.1.0", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -36,9 +36,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.0.4", - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/data-mate": "^1.1.0", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "awesome-phonenumber": "^2.70.0", "graphlib": "^2.1.8", "is-ip": "^3.1.0", diff --git a/packages/types/package.json b/packages/types/package.json index 4af7f936442..3c3f22fb26b 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/types", "displayName": "Types", - "version": "1.0.1", + "version": "1.1.0", "description": "A collection of typescript interfaces", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/types#readme", "bugs": { diff --git a/packages/types/src/teraslice.ts b/packages/types/src/teraslice.ts index 4c31544eef9..c425dc6e6fe 100644 --- a/packages/types/src/teraslice.ts +++ b/packages/types/src/teraslice.ts @@ -27,15 +27,12 @@ export interface AssetUploadQuery { } export interface JobSearchParams extends APISearchParams { - status: SearchJobStatus; + deleted?: boolean; + active?: boolean; } export type SearchQuery = APISearchParams & Record; -export type JobListStatusQuery = SearchJobStatus | JobSearchParams; - -export type SearchJobStatus = '*' | ExecutionStatus; - export interface APISearchParams { size?: number; from?: number; @@ -68,6 +65,8 @@ export interface JobConfig extends ValidatedJobConfig { _context: 'job'; _created: string | Date; _updated: string | Date; + _deleted?: boolean; + _deleted_on?: string | Date; } export enum RecoveryCleanupType { @@ -82,6 +81,8 @@ export interface ExecutionConfig extends ValidatedJobConfig { _context: 'ex'; _created: string | Date; _updated: string | Date; + _deleted?: boolean; + _deleted_on?: string | Date; // TODO: fix this metadata: Record; recovered_execution?: string; diff --git a/packages/utils/package.json b/packages/utils/package.json index 14d2bd81f83..aa72e94120b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "1.0.1", + "version": "1.1.0", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { @@ -29,7 +29,7 @@ "debug": "^4.3.6" }, "dependencies": { - "@terascope/types": "^1.0.1", + "@terascope/types": "^1.1.0", "@turf/bbox": "^6.4.0", "@turf/bbox-polygon": "^6.4.0", "@turf/boolean-contains": "^6.4.0", diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index c4ee4326548..0b7e46af1b0 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "1.0.3", + "version": "1.1.0", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -33,8 +33,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "peggy": "~4.0.3", "ts-pegjs": "^4.2.1" }, diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index ee88db4913c..1cb384ad9fc 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "1.0.3", + "version": "1.1.0", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -29,10 +29,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^1.0.1", - "@terascope/utils": "^1.0.1", + "@terascope/types": "^1.1.0", + "@terascope/utils": "^1.1.0", "@types/elasticsearch": "^5.0.43", - "xlucene-parser": "^1.0.3" + "xlucene-parser": "^1.1.0" }, "devDependencies": { "elasticsearch": "^15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index ba539f64478..e7a01e0d221 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "1.0.1", + "version": "1.1.0", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -24,10 +24,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^1.0.1" + "@terascope/utils": "^1.1.0" }, "devDependencies": { - "@terascope/types": "^1.0.1" + "@terascope/types": "^1.1.0" }, "engines": { "node": ">=18.18.0",