diff --git a/.travis.yml b/.travis.yml index 4fd90823d21..da7cd64abaf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,6 @@ jobs: if: branch = master script: yarn test --suite unit after_success: - - bash <(curl -s https://codecov.io/bash) -cF unit # verify the ui packages can build - yarn --cwd=./packages/ui-core build - yarn --cwd=./packages/ui-data-access build @@ -45,24 +44,22 @@ jobs: name: 'ES Test Suite (elasticsearch 5) (node 10)' # run only on pull-requests if: branch = master AND type IN (pull_request) AND fork = false - script: yarn test --suite elasticsearch --elasticsearch-version 5.6 --elasticsearch-api-version 6.5 + script: yarn test --suite elasticsearch --elasticsearch-version 5.6 --elasticsearch-api-version 6.5 --report-coverage false - script: name: 'ES Test Suite (elasticsearch 6) (node 10)' # run only on pull-requests and cron if: branch = master AND type IN (pull_request, cron) AND fork = false script: yarn test --suite elasticsearch --elasticsearch-version 6.8 --elasticsearch-api-version 6.5 - # only report coverage on elasticsearch@6 - after_success: bash <(curl -s https://codecov.io/bash) -cF elasticsearch - script: name: 'ES Test Suite (elasticsearch 7) (node 10)' # run only on pull-requests if: branch = master AND type IN (pull_request) AND fork = false - script: yarn test --suite elasticsearch --elasticsearch-version 7.2 --elasticsearch-api-version 7.0 + script: yarn test --suite elasticsearch --elasticsearch-version 7.2 --elasticsearch-api-version 7.0 --report-coverage false - script: - name: 'E2E Test Suite (elasticsearch 6) (node 10)' + name: 'End-to-End Test Suite (elasticsearch 6) (node 10)' # run only on pull-requests and cron if: branch = master AND type IN (pull_request, cron) AND fork = false script: yarn test --suite e2e --elasticsearch-version 6.8 --elasticsearch-api-version 6.5 --kafka-version 2.1 @@ -96,12 +93,13 @@ jobs: script: yarn docs deploy: - provider: script - script: ./scripts/docker-release.sh tag + skip_cleanup: true + script: ./scripts/publish-packages.sh on: all_branches: true tags: true - provider: script - script: ./scripts/publish-packages.sh + script: ./scripts/docker-release.sh tag on: all_branches: true tags: true diff --git a/docs/jobs/configuration.md b/docs/jobs/configuration.md index 17a09346efc..519683f2922 100644 --- a/docs/jobs/configuration.md +++ b/docs/jobs/configuration.md @@ -10,18 +10,19 @@ Note that the job configuration is divided into top level job configuration, and The first operation in the [operations](#operations) list, reads from a particular source, [see "Reader"](./types-of-operations.md#readers). The "Reader" will creates [Slices](../packages/job-components/api/interfaces/slice.md) which goes through the pipeline of operations specified on the job. -| Configuration | Description | Type | Notes | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Name for the given job | `String` | optional | -| `lifecycle` | Determines system exiting behaviour. Set to either **"once"** which will run the job to completion then the process will exit or **"persistent"** in which the process will continue until it is shutdown manually. | `String` | required | -| `analytics` | Determines if analytics should be ran for each slice | `Boolean` | optional, defaults to true | -| `max_retries` | Number of times a given slice of data will attempt to process before continuing on | `Number` | optional | -| `slicers` | Number of slicer functions that will chunk and prep the data for worker | `Number` | optional, defaults to 1 | -| `workers` | Number of worker instances that will process data, depending on the nature of the operations you may choose to over subscribe the number of workers compared to the number of cpu's | Number | optional, defaults to 5, if the number of workers for the job is set above workers specified in system configuration, a warning is passed and the workers set in the system configuration will be used. | -| `assets` | An array of strings that are the id's for the corresponding assets zip files. | `Array` | optional | -| `operations` | An array containing all the [operations](#operations) as well as their configurations. Typically the first is the reader/slicer. | `Array` | required | -| `apis` | An array containing all the [apis](#apis) as well as their configurations. | `Array` | required | -| `probation_window` | time in ms that the execution controller checks for failed slices, if there are none then it updates the state of the execution to running (this is only when lifecycle is set to persistent) | `Number` | optional | +| Configuration | Description | Type | Notes | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name for the given job | `String` | optional | +| `lifecycle` | Determines system exiting behaviour. Set to either **"once"** which will run the job to completion then the process will exit or **"persistent"** in which the process will continue until it is shutdown manually. | `String` | required | +| `analytics` | Determines if analytics should be ran for each slice | `Boolean` | optional, defaults to true | +| `performance_metrics` | Determines if performance metrics to collect garbage collector and other system stats should be collected | `Boolean` | optional, defaults to false | +| `max_retries` | Number of times a given slice of data will attempt to process before continuing on | `Number` | optional | +| `slicers` | Number of slicer functions that will chunk and prep the data for worker | `Number` | optional, defaults to 1 | +| `workers` | Number of worker instances that will process data, depending on the nature of the operations you may choose to over subscribe the number of workers compared to the number of cpu's | Number | optional, defaults to 5, if the number of workers for the job is set above workers specified in system configuration, a warning is passed and the workers set in the system configuration will be used. | +| `assets` | An array of strings that are the id's for the corresponding assets zip files. | `Array` | optional | +| `operations` | An array containing all the [operations](#operations) as well as their configurations. Typically the first is the reader/slicer. | `Array` | required | +| `apis` | An array containing all the [apis](#apis) as well as their configurations. | `Array` | required | +| `probation_window` | time in ms that the execution controller checks for failed slices, if there are none then it updates the state of the execution to running (this is only when lifecycle is set to persistent) | `Number` | optional | ### operations diff --git a/e2e/test/cases/cluster/state-spec.js b/e2e/test/cases/cluster/state-spec.js index 18cb57b8b20..e3ea5745269 100644 --- a/e2e/test/cases/cluster/state-spec.js +++ b/e2e/test/cases/cluster/state-spec.js @@ -99,8 +99,8 @@ describe('cluster state', () => { jobSpec.operations[1].index = specIndex; const job = await submitAndStart(jobSpec, 1000); - await Promise.delay(1000); const jobId = job.id(); + const state = await teraslice.cluster.state(); const complete = waitForJobStatus(job, 'completed'); diff --git a/jest.config.base.js b/jest.config.base.js index 0f09954f797..9a6805096f5 100644 --- a/jest.config.base.js +++ b/jest.config.base.js @@ -10,7 +10,6 @@ module.exports = (projectDir) => { const rootDir = name === 'e2e' ? '../' : '../../'; const packageRoot = name === 'e2e' ? '/e2e' : `/${workspaceName}/${name}`; const isTypescript = fs.pathExistsSync(path.join(projectDir, 'tsconfig.json')); - const runInPackage = projectDir === process.cwd(); const config = { rootDir, @@ -55,19 +54,11 @@ module.exports = (projectDir) => { }; if (isTypescript) { - if (runInPackage) { - config.globals['ts-jest'] = { - tsConfig: './tsconfig.json', - diagnostics: true, - pretty: true - }; - } else { - config.globals['ts-jest'] = { - tsConfig: `./${workspaceName}/${name}/tsconfig.json`, - diagnostics: true, - pretty: true - }; - } + config.globals['ts-jest'] = { + tsConfig: `./${workspaceName}/${name}/tsconfig.json`, + diagnostics: true, + pretty: true + }; } else { config.globals['ts-jest'] = { diagnostics: true, diff --git a/package.json b/package.json index 68d69f76da2..828a91a33c3 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "jest-watch-typeahead": "^0.3.1", "lerna": "^3.15.0", "lerna-alias": "^3.0.2", - "markdown-table": "^1.1.3", "pkg-up": "^3.1.0", "semver": "^6.1.1", "ts-jest": "^24.0.2", diff --git a/packages/data-access-plugin/package.json b/packages/data-access-plugin/package.json index d6b54579b89..d22977f5a54 100644 --- a/packages/data-access-plugin/package.json +++ b/packages/data-access-plugin/package.json @@ -28,21 +28,23 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { + "@apollographql/graphql-playground-html": "^1.6.24", "@terascope/data-access": "^0.12.6", "@terascope/data-types": "^0.5.2", "@terascope/elasticsearch-api": "^2.1.1", "@terascope/utils": "^0.14.1", + "accepts": "^1.3.7", "apollo-server-express": "^2.6.5", + "body-parser": "1.19.0", "elasticsearch-store": "^0.10.3", "graphql": "^14.3.1", "graphql-iso-date": "^3.6.1", - "graphql-tag": "^2.10.1", "graphql-type-json": "^0.3.0", "terafoundation": "^0.11.2", "xlucene-evaluator": "^0.9.6" }, "devDependencies": { - "@terascope/job-components": "^0.20.7", + "@terascope/job-components": "^0.20.8", "@types/express": "^4.17.0", "@types/got": "^9.6.4", "@types/graphql-iso-date": "^3.3.1", diff --git a/packages/job-components/package.json b/packages/job-components/package.json index dc87bc5a6fa..1e9957fcac4 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,6 +1,6 @@ { "name": "@terascope/job-components", - "version": "0.20.7", + "version": "0.20.8", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { diff --git a/packages/job-components/src/interfaces/jobs.ts b/packages/job-components/src/interfaces/jobs.ts index bbb2c6af8d3..cc418b61b27 100644 --- a/packages/job-components/src/interfaces/jobs.ts +++ b/packages/job-components/src/interfaces/jobs.ts @@ -65,6 +65,7 @@ export interface ValidatedJobConfig { apis: APIConfig[]; operations: OpConfig[]; probation_window: number; + performance_metrics?: boolean; slicers: number; workers: number; /** This will only be available in the context of k8s */ diff --git a/packages/job-components/src/job-schemas.ts b/packages/job-components/src/job-schemas.ts index 37c2a425717..bc97be90fcd 100644 --- a/packages/job-components/src/job-schemas.ts +++ b/packages/job-components/src/job-schemas.ts @@ -15,6 +15,11 @@ export function jobSchema(context: Context): convict.Schema { doc: 'logs the time it took in milliseconds for each action, ' + 'as well as the number of docs it receives', format: Boolean, }, + performance_metrics: { + default: false, + doc: 'logs performance metrics, including gc, loop and usage metrics for nodejs', + format: Boolean, + }, assets: { default: null, doc: diff --git a/packages/job-components/test/config-validators-spec.ts b/packages/job-components/test/config-validators-spec.ts index 3eb1b34d517..b42e8c542fb 100644 --- a/packages/job-components/test/config-validators-spec.ts +++ b/packages/job-components/test/config-validators-spec.ts @@ -27,12 +27,13 @@ describe('when using native clustering', () => { apis: [], operations: [{ _op: 'noop' }, { _op: 'noop' }], probation_window: 300000, + performance_metrics: false, slicers: 1, }; const jobConfig = validateJobConfig(schema, job); delete jobConfig.workers; - expect(jobConfig as object).toEqual(validJob); + expect(jobConfig).toEqual(validJob); }); }); @@ -296,7 +297,7 @@ describe('when using native clustering', () => { }; const config = validateOpConfig(schema, op); - expect(config as object).toEqual({ + expect(config).toEqual({ _op: 'some-op', _encoding: 'json', _dead_letter_action: 'none', @@ -315,7 +316,7 @@ describe('when using native clustering', () => { }; const config = validateOpConfig(schema, op); - expect(config as object).toEqual({ + expect(config).toEqual({ _op: 'some-op', _encoding: 'json', _dead_letter_action: 'none', @@ -361,7 +362,7 @@ describe('when using native clustering', () => { }; const config = validateOpConfig(schema, op); - expect(config as object).toEqual({ + expect(config).toEqual({ _op: 'some-op', _encoding: 'json', _dead_letter_action: 'log', @@ -433,7 +434,7 @@ describe('when using native clustering', () => { }; const config = validateAPIConfig(schema, api); - expect(config as object).toEqual({ + expect(config).toEqual({ _name: 'some-api', example: 'example', formatted_value: 'hi', @@ -510,6 +511,7 @@ describe('when validating k8s clustering', () => { apis: [], operations: [{ _op: 'noop' }, { _op: 'noop' }], probation_window: 300000, + performance_metrics: false, slicers: 1, targets: [ { @@ -522,7 +524,7 @@ describe('when validating k8s clustering', () => { const jobConfig = validateJobConfig(schema, job); delete jobConfig.workers; - expect(jobConfig as object).toEqual(validJob); + expect(jobConfig).toEqual(validJob); }); }); @@ -554,6 +556,7 @@ describe('when validating k8s clustering', () => { apis: [], operations: [{ _op: 'noop' }, { _op: 'noop' }], probation_window: 300000, + performance_metrics: false, targets: [], volumes: [ { @@ -566,7 +569,7 @@ describe('when validating k8s clustering', () => { const jobConfig = validateJobConfig(schema, job); delete jobConfig.workers; - expect(jobConfig as object).toEqual(validJob); + expect(jobConfig).toEqual(validJob); }); }); }); diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 9d4e349575d..2170cccd918 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,6 +1,6 @@ { "name": "@terascope/scripts", - "version": "0.1.1", + "version": "0.1.2", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -21,7 +21,6 @@ "scripts": { "build": "tsc --build --pretty", "build:watch": "yarn build --watch", - "postinstall": "./scripts/build-typedoc.js", "lint": "tslint -p tsconfig.json -t verbose -e '**/*.json'", "lint:fix": "yarn lint --fix", "prepublishOnly": "yarn build", @@ -33,18 +32,18 @@ "@lerna/query-graph": "^3.14.0", "@terascope/utils": "^0.14.1", "@types/is-ci": "^2.0.0", + "codecov": "^3.5.0", "execa": "^2.0.3", "fs-extra": "^8.1.0", "got": "^9.6.0", "ip": "^1.1.5", "is-ci": "^2.0.0", - "markdown-table": "^1.1.3", "ms": "^2.1.2", "pkg-up": "^3.1.0", "semver": "^6.1.1", "signale": "^1.4.0", "sort-package-json": "^1.22.1", - "typedoc": "TypeStrong/typedoc#07afa11", + "typedoc": "^0.15.0", "typedoc-plugin-markdown": "^2.0.7", "yargs": "^13.2.4" }, diff --git a/packages/scripts/scripts/build-typedoc.js b/packages/scripts/scripts/build-typedoc.js deleted file mode 100755 index bdd5ded5440..00000000000 --- a/packages/scripts/scripts/build-typedoc.js +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node - -'use strict'; - -process.env.NODE_ENV = 'development'; -process.env.FORCE_COLOR = '1'; - -const path = require('path'); -const fse = require('fs-extra'); -const execa = require('execa'); - -try { - require('typedoc'); -} catch (_) { - buildTypeDoc() - .then(() => { - require('typedoc'); - process.exit(0); - }) - .catch((err) => { - // eslint-disable-next-line no-console - console.error(err.message, err.all); - process.exit(1); - }); -} - -async function buildTypeDoc() { - const typeDocPath = path.dirname(require.resolve('typedoc/package.json')); - const pkgJSONPath = path.join(typeDocPath, 'package.json'); - const pkgJSON = await fse.readJSON(pkgJSONPath); - for (const dep of Object.keys(pkgJSON.devDependencies)) { - if (dep.includes('mocha') || dep.includes('mockery') || dep === 'nyc' || dep === 'tslint') { - delete pkgJSON.devDependencies[dep]; - } - } - await fse.writeJSON(pkgJSONPath, pkgJSON); - await fse.remove(path.join(typeDocPath, 'package-lock.json')); - await fse.emptyDir(path.join(typeDocPath, 'src', 'test')); - await execa('yarn', ['install'], { - cwd: typeDocPath - }); -} diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index c20c88c1306..ebd1f81829a 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -12,6 +12,7 @@ type Options = { watch: boolean; bail: boolean; suite?: TestSuite; + 'report-coverage': boolean; 'elasticsearch-host': string; 'elasticsearch-version': string; 'elasticsearch-api-version': string; @@ -37,6 +38,11 @@ const cmd: CommandModule = { type: 'boolean', default: isCI, }) + .option('report-coverage', { + description: 'Report the coverage for CI', + type: 'boolean', + default: isCI, + }) .option('watch', { description: 'Run tests in an interactive watch mode, this will test only the changed files', type: 'boolean', @@ -103,6 +109,7 @@ const cmd: CommandModule = { kafkaBroker: argv['kafka-broker'], kafkaVersion: argv['kafka-version'], all: !argv.packages || !argv.packages.length, + reportCoverage: argv['report-coverage'], jestArgs, }); }, diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index e22f041a433..cd19ea8f587 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -103,19 +103,25 @@ async function runTestSuite(suite: TestSuite, pkgInfos: PackageInfo[], options: } if (!errors.length) { + // jest or our tests have a memory leak, limiting this to 5 seems to help const chunked = chunk(pkgInfos, options.debug ? 1 : 5); const timeLabel = `test suite "${suite}"`; signale.time(timeLabel); + const env = utils.getEnv(options); if (options.debug || isCI) { signale.debug(`setting env for test suite "${suite}"`, env); } + let chunkIndex = -1; for (const pkgs of chunked) { - if (pkgs.length > 1) { - writePkgHeader('Running batch of tests', pkgs, true); - } else { + chunkIndex++; + + if (!pkgs.length) continue; + if (pkgs.length === 1) { writePkgHeader('Running test', pkgs, true); + } else { + writeHeader(`Running batch of ${pkgs.length} tests`, true); } const args = utils.getArgs(options); @@ -141,6 +147,10 @@ async function runTestSuite(suite: TestSuite, pkgInfos: PackageInfo[], options: if (options.bail) { break; } + } finally { + if (options.reportCoverage) { + await utils.reportCoverage(suite, chunkIndex); + } } } diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index 7c98ffbc5e5..96eef507d8d 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -5,6 +5,7 @@ export type TestOptions = { debug: boolean; watch: boolean; all: boolean; + reportCoverage: boolean; suite?: TestSuite; elasticsearchHost: string; elasticsearchVersion: string; diff --git a/packages/scripts/src/helpers/test-runner/utils.ts b/packages/scripts/src/helpers/test-runner/utils.ts index 3cb7a74b05e..8bc89af5ae2 100644 --- a/packages/scripts/src/helpers/test-runner/utils.ts +++ b/packages/scripts/src/helpers/test-runner/utils.ts @@ -3,7 +3,7 @@ import path from 'path'; import isCI from 'is-ci'; import fse from 'fs-extra'; import { debugLogger, get, TSError, isFunction } from '@terascope/utils'; -import { ArgsMap, ExecEnv, dockerBuild, dockerPull, exec } from '../scripts'; +import { ArgsMap, ExecEnv, dockerBuild, dockerPull, exec, fork } from '../scripts'; import { TestOptions, GroupedPackages } from './interfaces'; import { PackageInfo, TestSuite } from '../interfaces'; import { HOST_IP } from '../config'; @@ -193,7 +193,7 @@ export async function logE2E(dir: string, failed: boolean): Promise { RAW_LOGS: 'true', }); - const logFilePath = path.join(dir, './teraslice-test.log'); + const logFilePath = path.join(dir, 'teraslice-test.log'); if (!rawLogs) { await fse.remove(logFilePath); return; @@ -202,3 +202,19 @@ export async function logE2E(dir: string, failed: boolean): Promise { await fse.writeFile(logFilePath, rawLogs); signale.debug(`Wrote e2e log files to ${path.relative(process.cwd(), logFilePath)}`); } + +const abc = 'abcdefghijklmnopqrstuvwxyz'; + +export async function reportCoverage(suite: TestSuite, chunkIndex: number) { + const id = abc[chunkIndex] || 'any'; + + signale.info('* reporting coverage'); + try { + await fork({ + cmd: 'codecov', + args: ['--clear', '--flags', `${suite}-${id}`], + }); + } catch (err) { + signale.error(err); + } +} diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 78ddedc68c3..19c0ce0ca15 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -33,10 +33,11 @@ "lodash": "^4.17.11", "node-yaml": "^4.0.0", "prompts": "^2.1.0", - "syncprompt": "^2.0.0", "teraslice-client-js": "^0.7.2", "tmp": "^0.1.0", "tty-table": "^2.7.0", + "request": "^2.88.0", + "request-promise": "^4.2.4", "yargs": "^13.2.4", "yeoman-environment": "^2.3.4", "yeoman-generator": "^4.0.1", diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json index ec72c8a9673..d1e452c32f2 100644 --- a/packages/teraslice-op-test-harness/package.json +++ b/packages/teraslice-op-test-harness/package.json @@ -17,7 +17,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/job-components": "^0.20.7", + "@terascope/job-components": "^0.20.8", "bluebird": "^3.5.5", "lodash": "^4.17.11" }, diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index ce3ec4b14d0..49a228c363d 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -26,7 +26,7 @@ }, "dependencies": { "@terascope/elasticsearch-api": "^2.1.1", - "@terascope/job-components": "^0.20.7", + "@terascope/job-components": "^0.20.8", "bluebird": "^3.5.5", "mnemonist": "^0.30.0" }, diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index c839dd92c65..3bfd2eed928 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -32,7 +32,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/job-components": "^0.20.7", + "@terascope/job-components": "^0.20.8", "@terascope/teraslice-op-test-harness": "^1.7.1", "lodash": "^4.17.11" }, diff --git a/packages/teraslice/lib/workers/execution-controller/index.js b/packages/teraslice/lib/workers/execution-controller/index.js index 4d9c4ba0900..792f6b10aec 100644 --- a/packages/teraslice/lib/workers/execution-controller/index.js +++ b/packages/teraslice/lib/workers/execution-controller/index.js @@ -4,15 +4,15 @@ const _ = require('lodash'); const pWhilst = require('p-whilst'); const Messaging = require('@terascope/teraslice-messaging'); const { - TSError, get, pDelay, getFullErrorStack + TSError, get, pDelay, getFullErrorStack, isTest } = require('@terascope/utils'); - -const Scheduler = require('./scheduler'); -const ExecutionAnalytics = require('./execution-analytics'); -const makeSliceAnalytics = require('./slice-analytics'); const { waitForWorkerShutdown } = require('../helpers/worker-shutdown'); const { makeStateStore, makeExStore } = require('../../cluster/storage'); const { makeLogger, generateWorkerId } = require('../helpers/terafoundation'); +const ExecutionAnalytics = require('./execution-analytics'); +const makeSliceAnalytics = require('./slice-analytics'); +const Scheduler = require('./scheduler'); +const Metrics = require('../metrics'); const ExecutionControllerServer = Messaging.ExecutionController.Server; const ClusterMasterClient = Messaging.ClusterMaster.Client; @@ -24,6 +24,7 @@ class ExecutionController { const logger = makeLogger(context, 'execution_controller'); const events = context.apis.foundation.getSystemEvents(); const slicerPort = executionContext.config.slicer_port; + const performanceMetrics = executionContext.config.performance_metrics; const config = context.sysconfig.teraslice; const networkLatencyBuffer = get(config, 'network_latency_buffer'); const actionTimeout = get(config, 'action_timeout'); @@ -55,6 +56,20 @@ class ExecutionController { this.executionAnalytics = new ExecutionAnalytics(context, executionContext, this.client); this.scheduler = new Scheduler(context, executionContext); + try { + this.metrics = new Metrics({ + enabled: performanceMetrics, + logger, + statsInterval: isTest ? 500 : 5000 + }); + } catch (err) { + // this is a non-fatal error + logger.error( + new TSError(err, { + reason: 'Failure constructing metrics' + }) + ); + } this.exId = executionContext.exId; this.workerId = workerId; @@ -127,6 +142,19 @@ class ExecutionController { await this.server.start(); + try { + if (this.metrics != null) { + await this.metrics.initialize(); + } + } catch (err) { + // this is a non-fatal error + this.logger.error( + new TSError(err, { + reason: 'Failure initializing metrics' + }) + ); + } + this.isInitialized = true; this.server.onClientOnline((workerId) => { @@ -407,6 +435,10 @@ class ExecutionController { (async () => { const stores = Object.values(this.stores); await Promise.all(stores.map(store => store.shutdown(true).catch(pushError))); + })(), + (async () => { + if (this.metrics == null) return; + await this.metrics.shutdown().catch(pushError); })() ]); diff --git a/packages/teraslice/lib/workers/metrics/index.js b/packages/teraslice/lib/workers/metrics/index.js new file mode 100644 index 00000000000..464a8da72c5 --- /dev/null +++ b/packages/teraslice/lib/workers/metrics/index.js @@ -0,0 +1,132 @@ +/* eslint-disable no-console */ + +'use strict'; + +const { EventEmitter } = require('events'); +const { makeISODate, pDelay, debugLogger } = require('@terascope/utils'); + +const _logger = debugLogger('metrics'); + +class Metrics extends EventEmitter { + constructor({ enabled = true, logger, statsInterval = 5000 }) { + super(); + this.enabled = enabled; + this.logger = logger + ? logger.child({ + module: 'performance:metrics' + }) + : _logger; + this.statsInterval = statsInterval; + this.intervals = []; + if (this.enabled) { + this.emitter = require('@newrelic/native-metrics')({ + timeout: statsInterval + }); + this.logger.info('performance metrics are enabled'); + } else { + this.logger.info('performance metrics are disabled'); + } + } + + async initialize() { + if (!this.enabled) return; + + const { gcEnabled, loopEnabled, usageEnabled } = this.emitter; + this.logger.info('starting performance metrics', { + gcEnabled, + loopEnabled, + usageEnabled + }); + + if (gcEnabled) { + this.intervals.push( + setInterval(() => { + this._emitGCMetrics(); + }, this.statsInterval) + ); + } + + if (usageEnabled) { + this.emitter.on('usage', (usage) => { + this._emitUsageMetrics(usage); + }); + } + + if (loopEnabled) { + this.intervals.push( + setInterval(() => { + this._emitLoopMetrics(); + }, this.statsInterval) + ); + } + } + + async shutdown() { + this.intervals.forEach(clearInterval); + this.intervals = []; + if (this.emitter != null) { + this.emitter.unbind(); + } + this.removeAllListeners(); + } + + _emitUsageMetrics(usage) { + this._emitMetric('usage', usage); + } + + _emitGCMetrics() { + const gcMetrics = this.emitter.getGCMetrics(); + for (const [type, metric] of Object.entries(gcMetrics)) { + this._emitMetric('gc', { + type_id: metric.type_name, + type_name: type, + metrics: metric.metrics + }); + } + } + + _emitLoopMetrics() { + const loopMetrics = this.emitter.getLoopMetrics(); + this._emitMetric('loop', { + total_time: loopMetrics.usage.total, + min_time: loopMetrics.usage.min, + max_time: loopMetrics.usage.max, + sum_of_squares: loopMetrics.usage.sumOfSquares, + count: loopMetrics.usage.count + }); + } + + _emitMetric(type, data) { + const msg = { + type, + timestamp: makeISODate(), + data + }; + this.emit('metric', msg); + this.logger.info(msg, `performance metrics for type ${type}`); + } +} + +if (require.main === module) { + (async () => { + const metrics = new Metrics(); + metrics.on('metric', (metric) => { + console.dir(metric); + }); + + try { + await metrics.initialize(); + console.log('staying alive for 30s...'); + await pDelay(30000); + console.log('done, exiting...'); + } catch (err) { + console.error(err); + process.exitCode = 1; + } finally { + await metrics.shutdown(); + process.exit(); + } + })(); +} else { + module.exports = Metrics; +} diff --git a/packages/teraslice/lib/workers/worker/index.js b/packages/teraslice/lib/workers/worker/index.js index f251ac759f3..559e1a0778a 100644 --- a/packages/teraslice/lib/workers/worker/index.js +++ b/packages/teraslice/lib/workers/worker/index.js @@ -1,10 +1,13 @@ 'use strict'; -const { get, getFullErrorStack, isFatalError } = require('@terascope/utils'); const { ExecutionController, formatURL } = require('@terascope/teraslice-messaging'); +const { + get, getFullErrorStack, isFatalError, TSError, isTest +} = require('@terascope/utils'); const { makeStateStore, makeAnalyticsStore } = require('../../cluster/storage'); -const { waitForWorkerShutdown } = require('../helpers/worker-shutdown'); const { generateWorkerId, makeLogger } = require('../helpers/terafoundation'); +const { waitForWorkerShutdown } = require('../helpers/worker-shutdown'); +const Metrics = require('../metrics'); const Slice = require('./slice'); class Worker { @@ -15,7 +18,8 @@ class Worker { const { slicer_port: slicerPort, - slicer_hostname: slicerHostname + slicer_hostname: slicerHostname, + performance_metrics: performanceMetrics } = executionContext.config; const config = context.sysconfig.teraslice; @@ -35,6 +39,21 @@ class Worker { this.slice = new Slice(context, executionContext); + try { + this.metrics = new Metrics({ + enabled: performanceMetrics, + logger, + statsInterval: isTest ? 500 : 5000 + }); + } catch (err) { + // this is a non-fatal error + logger.error( + new TSError(err, { + reason: 'Failure constructing metrics' + }) + ); + } + this.stores = {}; this.executionContext = executionContext; this.shutdownTimeout = shutdownTimeout; @@ -55,8 +74,6 @@ class Worker { const { context } = this; this.isInitialized = true; - await this.executionContext.initialize(); - const stateStore = makeStateStore(context); const analyticsStore = makeAnalyticsStore(context); this.stores.stateStore = await stateStore; @@ -68,6 +85,22 @@ class Worker { }); await this.client.start(); + + // initialize the execution context next + await this.executionContext.initialize(); + + try { + if (this.metrics != null) { + await this.metrics.initialize(); + } + } catch (err) { + // this is a non-fatal error + this.logger.error( + new TSError(err, { + reason: 'Failure initializing metrics' + }) + ); + } } async run() { @@ -211,6 +244,10 @@ class Worker { })(), (async () => { await this.client.shutdown().catch(pushError); + })(), + (async () => { + if (this.metrics == null) return; + await this.metrics.shutdown().catch(pushError); })() ]); diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index ab42b27f687..272f82f30a9 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,6 +1,6 @@ { "name": "teraslice", - "version": "0.54.6", + "version": "0.54.7", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { @@ -32,9 +32,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { + "@newrelic/native-metrics": "^4.1.0", "@terascope/elasticsearch-api": "^2.1.1", "@terascope/error-parser": "^1.0.2", - "@terascope/job-components": "^0.20.7", + "@terascope/job-components": "^0.20.8", "@terascope/queue": "^1.1.6", "@terascope/teraslice-messaging": "^0.3.4", "@terascope/utils": "^0.14.1", diff --git a/packages/teraslice/test/workers/helpers/configs.js b/packages/teraslice/test/workers/helpers/configs.js index 398378fedc0..87a3b7ea3f5 100644 --- a/packages/teraslice/test/workers/helpers/configs.js +++ b/packages/teraslice/test/workers/helpers/configs.js @@ -72,6 +72,7 @@ const newConfig = (options = {}) => { lifecycle, max_retries: maxRetries, operations, + performance_metrics: false, recovered_execution: recoveredExecution, recovered_slice_type: recoveredSliceType, ex_id: newId('ex-id', true), diff --git a/packages/ui-components/package.json b/packages/ui-components/package.json index 426edfbe761..e0746c2d54d 100644 --- a/packages/ui-components/package.json +++ b/packages/ui-components/package.json @@ -23,7 +23,6 @@ "test": "echo '* tests not supported yet'" }, "dependencies": { - "@terascope/data-access": "^0.12.6", "@terascope/utils": "^0.14.1", "apollo-boost": "^0.4.3", "apollo-client": "^2.6.3", @@ -38,6 +37,7 @@ "semantic-ui-react": "^0.87.2" }, "devDependencies": { + "@terascope/data-access": "^0.12.6", "@types/jest": "24.0.15", "@types/node": "12.6.9", "@types/react": "16.8.23", diff --git a/packages/ui-components/src/SuccessMessage.tsx b/packages/ui-components/src/SuccessMessage.tsx index 59eda04e119..c761f8a9ddb 100644 --- a/packages/ui-components/src/SuccessMessage.tsx +++ b/packages/ui-components/src/SuccessMessage.tsx @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { Redirect } from 'react-router'; +import { Redirect } from 'react-router-dom'; import { Message, Icon } from 'semantic-ui-react'; const SuccessMessage: React.FC = ({ diff --git a/packages/ui-core/package.json b/packages/ui-core/package.json index aeee7b3cfa8..90d89dda1cd 100644 --- a/packages/ui-core/package.json +++ b/packages/ui-core/package.json @@ -53,6 +53,7 @@ "apollo-boost": "^0.4.3", "apollo-client": "^2.6.3", "graphql": "^14.3.1", + "graphql-tag": "^2.10.1", "history": "^4.9.0", "npm-watch": "^0.6.0", "prop-types": "^15.7.2", diff --git a/packages/ui-core/src/components/Logout.tsx b/packages/ui-core/src/components/Logout.tsx index 4500f59911f..f2b9cc91202 100644 --- a/packages/ui-core/src/components/Logout.tsx +++ b/packages/ui-core/src/components/Logout.tsx @@ -1,7 +1,7 @@ import React from 'react'; import gql from 'graphql-tag'; import { Query, ApolloConsumer } from 'react-apollo'; -import { Redirect } from 'react-router'; +import { Redirect } from 'react-router-dom'; import { parseErrorInfo } from '@terascope/utils'; import { LoadingPage, diff --git a/packages/ui-data-access/package.json b/packages/ui-data-access/package.json index 81de7db4e65..9da9424e255 100644 --- a/packages/ui-data-access/package.json +++ b/packages/ui-data-access/package.json @@ -35,10 +35,11 @@ "apollo-boost": "^0.4.3", "apollo-client": "^2.6.3", "css-loader": "^3.0.0", - "date-fns": "^1.30.1", "graphql": "^14.3.1", + "graphql-tag": "^2.10.1", "history": "^4.9.0", "prop-types": "^15.7.2", + "query-string": "^6.8.2", "react": "^16.8.6", "react-apollo": "^2.5.7", "react-dom": "^16.8.6", diff --git a/packages/utils/bench/data-entity-suite.js b/packages/utils/bench/data-entity-suite.js index 9f0f81cd559..a3d5b22ef36 100644 --- a/packages/utils/bench/data-entity-suite.js +++ b/packages/utils/bench/data-entity-suite.js @@ -2,10 +2,9 @@ /* eslint-disable no-unused-expressions */ -const { times } = require('@terascope/utils'); const { Suite } = require('./helpers'); const FakeDataEntity = require('./fixtures/fake-data-entity'); -const { DataEntity } = require('../dist/src'); +const { DataEntity, times } = require('../dist/src'); const data = {}; diff --git a/packages/xlucene-evaluator/package.json b/packages/xlucene-evaluator/package.json index 9a937e9ae8d..e3b6d116b61 100644 --- a/packages/xlucene-evaluator/package.json +++ b/packages/xlucene-evaluator/package.json @@ -43,8 +43,7 @@ "is-cidr": "^3.0.0", "latlon-geohash": "^1.1.0", "lodash": "^4.17.11", - "pegjs-backtrace": "^0.2.0", - "rambda": "^2.11.1" + "pegjs-backtrace": "^0.2.0" }, "devDependencies": { "elasticsearch": "^15.4.1", diff --git a/yarn.lock b/yarn.lock index de97da5d54e..e6388a837ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,7 +9,7 @@ dependencies: apollo-env "0.5.1" -"@apollographql/graphql-playground-html@1.6.24": +"@apollographql/graphql-playground-html@1.6.24", "@apollographql/graphql-playground-html@^1.6.24": version "1.6.24" resolved "https://registry.yarnpkg.com/@apollographql/graphql-playground-html/-/graphql-playground-html-1.6.24.tgz#3ce939cb127fb8aaa3ffc1e90dff9b8af9f2e3dc" integrity sha512-8GqG48m1XqyXh4mIZrtB5xOhUwSsh1WsrrsaZQOEYYql3YN9DEu9OOSg0ILzXHZo/h2Q74777YE4YzlArQzQEQ== @@ -1962,6 +1962,14 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@newrelic/native-metrics@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@newrelic/native-metrics/-/native-metrics-4.1.0.tgz#cb1641e55179cd6ec058441ae0b0eedfa36ee0eb" + integrity sha512-7CZlKMLuaYQW7mV9qVyo9b9HVe2xBnyn+kkETRJoZGs5P7gdfv9AAE3RPhtOBUopTfbmc8ju7njYadjui9J1XA== + dependencies: + nan "^2.12.1" + semver "^5.5.1" + "@nodelib/fs.stat@^1.1.2": version "1.1.3" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" @@ -2992,7 +3000,7 @@ accepts@1.3.3: mime-types "~2.1.11" negotiator "0.6.1" -accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: +accepts@^1.3.5, accepts@^1.3.7, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== @@ -3466,6 +3474,11 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" +argv@^0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/argv/-/argv-0.0.2.tgz#ecbd16f8949b157183711b1bda334f37840185ab" + integrity sha1-7L0W+JSbFXGDcRsb2jNPN4QBhas= + aria-query@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-3.0.0.tgz#65b3fcc1ca1155a8c9ae64d6eee297f15d5133cc" @@ -3923,7 +3936,7 @@ babylon@^6.18.0: resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" integrity sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ== -backbone@^1.1.2: +backbone@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/backbone/-/backbone-1.4.0.tgz#54db4de9df7c3811c3f032f34749a4cd27f3bd12" integrity sha512-RLmDrRXkVdouTg38jcgHhyQ/2zjg7a8E6sz2zxfz21Hh17xDJYUHBZimVIt5fUyS8vbfpeSmTL3gUjTEvUV3qQ== @@ -4051,13 +4064,6 @@ binaryextensions@^2.1.2: resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.1.2.tgz#c83c3d74233ba7674e4f313cb2a2b70f54e94b7c" integrity sha512-xVNN69YGDghOqCCtA6FI7avYrr02mTJjOgB0/f1VPD3pJC8QEvjTKWc4epDx8AqxxA75NI0QpVM2gPJXUbE4Tg== -bindings@^1.2.1: - version "1.5.0" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" - integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== - dependencies: - file-uri-to-path "1.0.0" - bl@^1.0.0: version "1.2.2" resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" @@ -4929,6 +4935,17 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +codecov@^3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/codecov/-/codecov-3.5.0.tgz#3d0748932f9cb41e1ad7f21fa346ef1b2b1bed47" + integrity sha512-/OsWOfIHaQIr7aeZ4pY0UC1PZT6kimoKFOFYFNb6wxo3iw12nRrh+mNGH72rnXxNsq6SGfesVPizm/6Q3XqcFQ== + dependencies: + argv "^0.0.2" + ignore-walk "^3.0.1" + js-yaml "^3.13.1" + teeny-request "^3.11.3" + urlgrey "^0.4.4" + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -7385,11 +7402,6 @@ file-type@^6.1.0: resolved "https://registry.yarnpkg.com/file-type/-/file-type-6.2.0.tgz#e50cd75d356ffed4e306dc4f5bcf52a79903a919" integrity sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg== -file-uri-to-path@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" - integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== - fileset@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/fileset/-/fileset-2.0.3.tgz#8e7548a96d3cc2327ee5e674168723a333bba2a0" @@ -7656,7 +7668,7 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@7.0.1, fs-extra@^7.0.0, fs-extra@^7.0.1: +fs-extra@7.0.1, fs-extra@^7.0.0: version "7.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9" integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw== @@ -8447,10 +8459,10 @@ hex-color-regex@^1.1.0: resolved "https://registry.yarnpkg.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz#4c06fccb4602fe2602b3c93df82d7e7dbf1a8a8e" integrity sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ== -highlight.js@^9.13.1: - version "9.15.8" - resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.15.8.tgz#f344fda123f36f1a65490e932cf90569e4999971" - integrity sha512-RrapkKQWwE+wKdF73VsOa2RQdIoO3mxwJ4P8mhbI6KYJUraUHRKM5w5zQQKXNk0xNL4UVRdulV9SBJcmzJNzVA== +highlight.js@^9.15.8: + version "9.15.9" + resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.15.9.tgz#865257da1dbb4a58c4552d46c4b3854f77f0e6d5" + integrity sha512-M0zZvfLr5p0keDMCAhNBp03XJbKBxUx5AfyfufMdFMEP4N/Xj6dh0IqC75ys7BAzceR34NgcvXjupRVaHBPPVQ== history@^4.9.0: version "4.9.0" @@ -10335,12 +10347,7 @@ joi@^14.3.1: isemail "3.x.x" topo "3.x.x" -jquery@^2.2.4: - version "2.2.4" - resolved "https://registry.yarnpkg.com/jquery/-/jquery-2.2.4.tgz#2c89d6889b5eac522a7eea32c14521559c6cbf02" - integrity sha1-LInWiJterFIqfuoywUUhVZxsvwI= - -jquery@x.*: +jquery@^3.4.1, jquery@x.*: version "3.4.1" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2" integrity sha512-36+AdBzCL+y6qjw5Tx7HgzeGCzC81MDDgaUP8ld2zhx58HdqXGoBd+tHdrBMiyjGQs0Hxs/MLZTu/eHNJJuWPw== @@ -10944,7 +10951,7 @@ lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= -"lodash@>=3.5 <5", lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.1, lodash@^4.3.0: +"lodash@>=3.5 <5", lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.1, lodash@^4.3.0: version "4.17.15" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== @@ -11116,15 +11123,10 @@ map-visit@^1.0.0: dependencies: object-visit "^1.0.0" -markdown-table@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-1.1.3.tgz#9fcb69bcfdb8717bfd0398c6ec2d93036ef8de60" - integrity sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q== - -marked@^0.6.2: - version "0.6.3" - resolved "https://registry.yarnpkg.com/marked/-/marked-0.6.3.tgz#79babad78af638ba4d522a9e715cdfdd2429e946" - integrity sha512-Fqa7eq+UaxfMriqzYLayfqAE40WN03jf+zHjT18/uXNuzjq3TY0XTbrAoPeqSJrAmPz11VuUA+kBPYOhHt9oOQ== +marked@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-0.7.0.tgz#b64201f051d271b1edc10a04d1ae9b74bb8e5c0e" + integrity sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg== md5.js@^1.3.4: version "1.3.5" @@ -11699,7 +11701,7 @@ mz@^2.5.0: object-assign "^4.0.1" thenify-all "^1.0.0" -nan@^2.10.0, nan@^2.4.0: +nan@^2.10.0: version "2.12.1" resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552" integrity sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw== @@ -14070,7 +14072,7 @@ query-string@^5.0.1: object-assign "^4.1.0" strict-uri-encode "^1.0.0" -query-string@^6.8.1: +query-string@^6.8.1, query-string@^6.8.2: version "6.8.2" resolved "https://registry.yarnpkg.com/query-string/-/query-string-6.8.2.tgz#36cb7e452ae11a4b5e9efee83375e0954407b2f6" integrity sha512-J3Qi8XZJXh93t2FiKyd/7Ec6GNifsjKXUsVFkSBj/kjLsDylWhnCz4NT1bkPcKotttPW+QbKGqqPH8OoI2pdqw== @@ -16145,14 +16147,6 @@ symbol-tree@^3.2.2: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.2.tgz#ae27db38f660a7ae2e1c3b7d1bc290819b8519e6" integrity sha1-rifbOPZgp64uHDt9G8KQgZuFGeY= -syncprompt@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/syncprompt/-/syncprompt-2.0.0.tgz#b4f303d385e6a0e214614a12d22e8d97234669a4" - integrity sha512-tPYCsLGWTDRlET7cen2iAwbmO+lSwfIdKoLkUPC41t/54UhqbzpwXL4RhODu5fQnmZdQZlEFKeD0W/BysU5WEg== - dependencies: - bindings "^1.2.1" - nan "^2.4.0" - table@^5.2.3: version "5.2.3" resolved "https://registry.yarnpkg.com/table/-/table-5.2.3.tgz#cde0cc6eb06751c009efab27e8c820ca5b67b7f2" @@ -16238,6 +16232,15 @@ tar@^4.4.10: safe-buffer "^5.1.2" yallist "^3.0.3" +teeny-request@^3.11.3: + version "3.11.3" + resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-3.11.3.tgz#335c629f7645e5d6599362df2f3230c4cbc23a55" + integrity sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw== + dependencies: + https-proxy-agent "^2.2.1" + node-fetch "^2.2.0" + uuid "^3.3.2" + temp-dir@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d" @@ -16767,13 +16770,13 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typedoc-default-themes@^0.6.0-0: - version "0.6.0-0" - resolved "https://registry.yarnpkg.com/typedoc-default-themes/-/typedoc-default-themes-0.6.0-0.tgz#a4867eaf91fb7888efd01680f1328b72e8a33640" - integrity sha512-O7hBMS1yBCozvVUntIIdlBk04WiqM+f6NOEc9p+LimJSFKJMF66cgzejeiybuTk6mgbMJW+olg42BNYC8E9x9Q== +typedoc-default-themes@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/typedoc-default-themes/-/typedoc-default-themes-0.6.0.tgz#7e73bf54dd9e11550dd0fb576d5176b758f8f8b5" + integrity sha512-MdTROOojxod78CEv22rIA69o7crMPLnVZPefuDLt/WepXqJwgiSu8Xxq+H36x0Jj3YGc7lOglI2vPJ2GhoOybw== dependencies: - backbone "^1.1.2" - jquery "^2.2.4" + backbone "^1.4.0" + jquery "^3.4.1" lunr "^2.3.6" underscore "^1.9.1" @@ -16782,20 +16785,21 @@ typedoc-plugin-markdown@^2.0.7: resolved "https://registry.yarnpkg.com/typedoc-plugin-markdown/-/typedoc-plugin-markdown-2.0.11.tgz#47d8aecd1667f9d083b0a3ebbff0918512808bd3" integrity sha512-8+0JR4RmZPk2oWp13bjFib1Idf9IXBxP4Bp610vWzybYeNR79aZbQu8k5HdJdj/kJrTvBzVFSdjA90C9UKWOxQ== -typedoc@TypeStrong/typedoc#07afa11: - version "0.15.0-0" - resolved "https://codeload.github.com/TypeStrong/typedoc/tar.gz/07afa11f859afed5b13cdad8d30122b387df1be8" +typedoc@^0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.15.0.tgz#21eaf4db41cf2797bad027a74f2a75cd08ae0c2d" + integrity sha512-NOtfq5Tis4EFt+J2ozhVq9RCeUnfEYMFKoU6nCXCXUULJz1UQynOM+yH3TkfZCPLzigbqB0tQYGVlktUWweKlw== dependencies: "@types/minimatch" "3.0.3" - fs-extra "^7.0.1" + fs-extra "^8.1.0" handlebars "^4.1.2" - highlight.js "^9.13.1" - lodash "^4.17.11" - marked "^0.6.2" + highlight.js "^9.15.8" + lodash "^4.17.15" + marked "^0.7.0" minimatch "^3.0.0" progress "^2.0.3" shelljs "^0.8.3" - typedoc-default-themes "^0.6.0-0" + typedoc-default-themes "^0.6.0" typescript "3.5.x" typescript@3.5.x, typescript@^3.5.2: @@ -17096,6 +17100,11 @@ url@^0.11.0: punycode "1.3.2" querystring "0.2.0" +urlgrey@^0.4.4: + version "0.4.4" + resolved "https://registry.yarnpkg.com/urlgrey/-/urlgrey-0.4.4.tgz#892fe95960805e85519f1cd4389f2cb4cbb7652f" + integrity sha1-iS/pWWCAXoVRnxzUOJ8stMu3ZS8= + use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"