Skip to content

Commit

Permalink
Merge pull request #1301 from terascope/add-more-metrics
Browse files Browse the repository at this point in the history 
v0.54.7 Add performance metrics
  • Loading branch information
@peterdemartini
peterdemartini authored Aug 5, 2019
2 parents 0db950f + 2bb2852 commit 695a6df
Show file tree
Hide file tree
Showing 33 changed files with 385 additions and 181 deletions.
14 changes: 6 additions & 8 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ jobs:
if: branch = master
script: yarn test --suite unit
after_success:
- bash <(curl -s https://codecov.io/bash) -cF unit
# verify the ui packages can build
- yarn --cwd=./packages/ui-core build
- yarn --cwd=./packages/ui-data-access build
Expand All @@ -45,24 +44,22 @@ jobs:
name: 'ES Test Suite (elasticsearch 5) (node 10)'
# run only on pull-requests
if: branch = master AND type IN (pull_request) AND fork = false
script: yarn test --suite elasticsearch --elasticsearch-version 5.6 --elasticsearch-api-version 6.5
script: yarn test --suite elasticsearch --elasticsearch-version 5.6 --elasticsearch-api-version 6.5 --report-coverage false

- script:
name: 'ES Test Suite (elasticsearch 6) (node 10)'
# run only on pull-requests and cron
if: branch = master AND type IN (pull_request, cron) AND fork = false
script: yarn test --suite elasticsearch --elasticsearch-version 6.8 --elasticsearch-api-version 6.5
# only report coverage on elasticsearch@6
after_success: bash <(curl -s https://codecov.io/bash) -cF elasticsearch

- script:
name: 'ES Test Suite (elasticsearch 7) (node 10)'
# run only on pull-requests
if: branch = master AND type IN (pull_request) AND fork = false
script: yarn test --suite elasticsearch --elasticsearch-version 7.2 --elasticsearch-api-version 7.0
script: yarn test --suite elasticsearch --elasticsearch-version 7.2 --elasticsearch-api-version 7.0 --report-coverage false

- script:
name: 'E2E Test Suite (elasticsearch 6) (node 10)'
name: 'End-to-End Test Suite (elasticsearch 6) (node 10)'
# run only on pull-requests and cron
if: branch = master AND type IN (pull_request, cron) AND fork = false
script: yarn test --suite e2e --elasticsearch-version 6.8 --elasticsearch-api-version 6.5 --kafka-version 2.1
Expand Down Expand Up @@ -96,12 +93,13 @@ jobs:
script: yarn docs
deploy:
- provider: script
script: ./scripts/docker-release.sh tag
skip_cleanup: true
script: ./scripts/publish-packages.sh
on:
all_branches: true
tags: true
- provider: script
script: ./scripts/publish-packages.sh
script: ./scripts/docker-release.sh tag
on:
all_branches: true
tags: true
Expand Down
25 changes: 13 additions & 12 deletions docs/jobs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,19 @@ Note that the job configuration is divided into top level job configuration, and

The first operation in the [operations](#operations) list, reads from a particular source, [see "Reader"](./types-of-operations.md#readers). The "Reader" will creates [Slices](../packages/job-components/api/interfaces/slice.md) which goes through the pipeline of operations specified on the job.

| Configuration | Description | Type | Notes |
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `name` | Name for the given job | `String` | optional |
| `lifecycle` | Determines system exiting behaviour. Set to either **"once"** which will run the job to completion then the process will exit or **"persistent"** in which the process will continue until it is shutdown manually. | `String` | required |
| `analytics` | Determines if analytics should be ran for each slice | `Boolean` | optional, defaults to true |
| `max_retries` | Number of times a given slice of data will attempt to process before continuing on | `Number` | optional |
| `slicers` | Number of slicer functions that will chunk and prep the data for worker | `Number` | optional, defaults to 1 |
| `workers` | Number of worker instances that will process data, depending on the nature of the operations you may choose to over subscribe the number of workers compared to the number of cpu's | Number | optional, defaults to 5, if the number of workers for the job is set above workers specified in system configuration, a warning is passed and the workers set in the system configuration will be used. |
| `assets` | An array of strings that are the id's for the corresponding assets zip files. | `Array` | optional |
| `operations` | An array containing all the [operations](#operations) as well as their configurations. Typically the first is the reader/slicer. | `Array` | required |
| `apis` | An array containing all the [apis](#apis) as well as their configurations. | `Array` | required |
| `probation_window` | time in ms that the execution controller checks for failed slices, if there are none then it updates the state of the execution to running (this is only when lifecycle is set to persistent) | `Number` | optional |
| Configuration | Description | Type | Notes |
| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `name` | Name for the given job | `String` | optional |
| `lifecycle` | Determines system exiting behaviour. Set to either **"once"** which will run the job to completion then the process will exit or **"persistent"** in which the process will continue until it is shutdown manually. | `String` | required |
| `analytics` | Determines if analytics should be ran for each slice | `Boolean` | optional, defaults to true |
| `performance_metrics` | Determines if performance metrics to collect garbage collector and other system stats should be collected | `Boolean` | optional, defaults to false |
| `max_retries` | Number of times a given slice of data will attempt to process before continuing on | `Number` | optional |
| `slicers` | Number of slicer functions that will chunk and prep the data for worker | `Number` | optional, defaults to 1 |
| `workers` | Number of worker instances that will process data, depending on the nature of the operations you may choose to over subscribe the number of workers compared to the number of cpu's | Number | optional, defaults to 5, if the number of workers for the job is set above workers specified in system configuration, a warning is passed and the workers set in the system configuration will be used. |
| `assets` | An array of strings that are the id's for the corresponding assets zip files. | `Array` | optional |
| `operations` | An array containing all the [operations](#operations) as well as their configurations. Typically the first is the reader/slicer. | `Array` | required |
| `apis` | An array containing all the [apis](#apis) as well as their configurations. | `Array` | required |
| `probation_window` | time in ms that the execution controller checks for failed slices, if there are none then it updates the state of the execution to running (this is only when lifecycle is set to persistent) | `Number` | optional |

### operations

Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/cluster/state-spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ describe('cluster state', () => {
jobSpec.operations[1].index = specIndex;

const job = await submitAndStart(jobSpec, 1000);
await Promise.delay(1000);
const jobId = job.id();

const state = await teraslice.cluster.state();

const complete = waitForJobStatus(job, 'completed');
Expand Down
19 changes: 5 additions & 14 deletions jest.config.base.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ module.exports = (projectDir) => {
const rootDir = name === 'e2e' ? '../' : '../../';
const packageRoot = name === 'e2e' ? '<rootDir>/e2e' : `<rootDir>/${workspaceName}/${name}`;
const isTypescript = fs.pathExistsSync(path.join(projectDir, 'tsconfig.json'));
const runInPackage = projectDir === process.cwd();

const config = {
rootDir,
Expand Down Expand Up @@ -55,19 +54,11 @@ module.exports = (projectDir) => {
};

if (isTypescript) {
if (runInPackage) {
config.globals['ts-jest'] = {
tsConfig: './tsconfig.json',
diagnostics: true,
pretty: true
};
} else {
config.globals['ts-jest'] = {
tsConfig: `./${workspaceName}/${name}/tsconfig.json`,
diagnostics: true,
pretty: true
};
}
config.globals['ts-jest'] = {
tsConfig: `./${workspaceName}/${name}/tsconfig.json`,
diagnostics: true,
pretty: true
};
} else {
config.globals['ts-jest'] = {
diagnostics: true,
Expand Down
1 change: 0 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
"jest-watch-typeahead": "^0.3.1",
"lerna": "^3.15.0",
"lerna-alias": "^3.0.2",
"markdown-table": "^1.1.3",
"pkg-up": "^3.1.0",
"semver": "^6.1.1",
"ts-jest": "^24.0.2",
Expand Down
6 changes: 4 additions & 2 deletions packages/data-access-plugin/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,23 @@
"test:watch": "ts-scripts test --watch . --"
},
"dependencies": {
"@apollographql/graphql-playground-html": "^1.6.24",
"@terascope/data-access": "^0.12.6",
"@terascope/data-types": "^0.5.2",
"@terascope/elasticsearch-api": "^2.1.1",
"@terascope/utils": "^0.14.1",
"accepts": "^1.3.7",
"apollo-server-express": "^2.6.5",
"body-parser": "1.19.0",
"elasticsearch-store": "^0.10.3",
"graphql": "^14.3.1",
"graphql-iso-date": "^3.6.1",
"graphql-tag": "^2.10.1",
"graphql-type-json": "^0.3.0",
"terafoundation": "^0.11.2",
"xlucene-evaluator": "^0.9.6"
},
"devDependencies": {
"@terascope/job-components": "^0.20.7",
"@terascope/job-components": "^0.20.8",
"@types/express": "^4.17.0",
"@types/got": "^9.6.4",
"@types/graphql-iso-date": "^3.3.1",
Expand Down
2 changes: 1 addition & 1 deletion packages/job-components/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@terascope/job-components",
"version": "0.20.7",
"version": "0.20.8",
"description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
"bugs": {
Expand Down
1 change: 1 addition & 0 deletions packages/job-components/src/interfaces/jobs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ export interface ValidatedJobConfig {
apis: APIConfig[];
operations: OpConfig[];
probation_window: number;
performance_metrics?: boolean;
slicers: number;
workers: number;
/** This will only be available in the context of k8s */
Expand Down
5 changes: 5 additions & 0 deletions packages/job-components/src/job-schemas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ export function jobSchema(context: Context): convict.Schema<any> {
doc: 'logs the time it took in milliseconds for each action, ' + 'as well as the number of docs it receives',
format: Boolean,
},
performance_metrics: {
default: false,
doc: 'logs performance metrics, including gc, loop and usage metrics for nodejs',
format: Boolean,
},
assets: {
default: null,
doc:
Expand Down
Loading

0 comments on commit 695a6df

Please sign in to comment.