Skip to content

Commit

Permalink
Merge pull request #867 from terascope/beam-me-up
Browse files Browse the repository at this point in the history
v0.43.0:  Add native support for the new Job APIs in teraslice
  • Loading branch information
godber authored Oct 31, 2018
2 parents 4603572 + c54d6ea commit c39e906
Show file tree
Hide file tree
Showing 157 changed files with 3,326 additions and 1,971 deletions.
9 changes: 8 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,21 @@ jobs:
# stop if given a reason
fast-finish: true
include:

# tests only on master and on pull-request
- stage: Tests
name: Test Packages
if: branch = master
script: yarn test
script: yarn test:ci
after_success:
- bash <(curl -s https://codecov.io/bash)

# benchmarks only on master and on pull-request
- script:
name: Benchmarks and Linting
if: branch = master
script: yarn lint && yarn benchmark

# test end-to-end tests, only on pull-requests
- script:
name: End-to-End Tests
Expand Down
2 changes: 1 addition & 1 deletion e2e/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"description": "Teraslice integration test suite",
"scripts": {
"test": "echo '[WARN] Make sure to remember to run yarn clean or yarn test:e2e:clean to remove the docker containers'; jest --runInBand",
"test:ci": "jest --runInBand $JEST_ARGS || (yarn logs; yarn clean; exit 1) && (yarn clean; exit 0)",
"test:ci": "jest --runInBand --bail=true || (yarn logs; yarn clean; exit 1) && (yarn clean; exit 0)",
"logs": "docker-compose logs --no-color teraslice-worker teraslice-master | awk -F' [|] ' '{print $2}' | bunyan -o short",
"logs-follow": "docker-compose logs --follow --no-color teraslice-worker teraslice-master | awk -F' [|] ' '{print $2}' | bunyan -o short",
"clean": "docker-compose down --volumes --remove-orphans --timeout=5"
Expand Down
1 change: 0 additions & 1 deletion e2e/test/cases/cluster/state-spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@ describe('cluster state', () => {
.then(() => misc.indexStats('test-clusterstate-job-1-1000')
.then((stats) => {
expect(stats.count).toBe(1000);
expect(stats.deleted).toBe(0);
}))
.catch((err) => {
fail(err);
Expand Down
1 change: 0 additions & 1 deletion e2e/test/cases/cluster/worker-allocation-spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ function workersTest(workers, workersExpected, records, done) {
.then(() => misc.indexStats(`test-allocation-${workers}-worker`)
.then((stats) => {
expect(stats.count).toBe(records);
expect(stats.deleted).toBe(0);
})))
.catch(fail)
.finally(() => {
Expand Down
3 changes: 0 additions & 3 deletions e2e/test/cases/data/id-reader-spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ describe('id reader', () => {
.then(() => misc.indexStats('test-id_reindex-10000')
.then((stats) => {
expect(stats.count).toBe(10000);
expect(stats.deleted).toBe(0);
}))
.catch(fail)
.finally(() => { done(); });
Expand All @@ -45,7 +44,6 @@ describe('id reader', () => {
.then(() => misc.indexStats('test-hexadecimal-logs')
.then((stats) => {
expect(stats.count).toBe(10000);
expect(stats.deleted).toBe(0);
}))
.catch(fail)
.finally(() => { done(); });
Expand Down Expand Up @@ -97,7 +95,6 @@ describe('id reader', () => {
.then(() => misc.indexStats('test-id_reindex-lifecycle-10000')
.then((stats) => {
expect(stats.count).toBe(10000);
expect(stats.deleted).toBe(0);
}));
})
.catch(fail)
Expand Down
5 changes: 2 additions & 3 deletions e2e/test/cases/data/reindex-spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ describe('reindex', () => {
})
.then((stats) => {
expect(stats.count).toBe(10);
expect(stats.deleted).toBe(0);
})
.catch(fail)
.finally(() => { done(); });
Expand Down Expand Up @@ -153,8 +152,8 @@ describe('reindex', () => {
})
.then(async () => {
const stats = await misc.indexStats(`test-reindex-${iterations}times`);
expect(stats.count).toBe(10 * iterations);
expect(stats.deleted).toBe(0);

expect(stats.count).toBe(10);
})
.catch(fail)
.finally(() => { done(); });
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/fixtures/jobs/multisend.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
{
"_op": "elasticsearch_reader",
"index": "example-logs-10000",
"full_response": true,
"preserve_id": true,
"date_field_name": "created",
"type": "change",
"size": 500
Expand Down
18 changes: 9 additions & 9 deletions e2e/test/fixtures/jobs/reindex.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,22 @@
"analytics": true,
"operations": [
{
"_op": "elasticsearch_reader",
"index": "example-logs-10",
"type": "change",
"size": 1000,
"date_field_name": "created",
"full_response": true
"_op": "elasticsearch_reader",
"index": "example-logs-10",
"type": "change",
"size": 10,
"date_field_name": "created",
"preserve_id": true
},
{
"_op": "elasticsearch_index_selector",
"type": "change",
"index": "example-logs-new",
"id_field": "_key"
"preserve_id": true
},
{
"_op": "elasticsearch_bulk",
"size": 100
"_op": "elasticsearch_bulk",
"size": 100
}
]
}
20 changes: 10 additions & 10 deletions examples/jobs/reindex.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,22 @@
"workers": 1,
"operations": [
{
"_op": "elasticsearch_reader",
"index": "example-logs",
"type": "logs",
"size": 10000,
"date_field_name": "created",
"full_response": true
"_op": "elasticsearch_reader",
"index": "example-logs",
"type": "logs",
"size": 10000,
"date_field_name": "created",
"preserve_id": true
},
{
"_op": "elasticsearch_index_selector",
"type": "change",
"index": "example-logs-new",
"id_field": "_key"
"preserve_id": true
},
{
"_op": "elasticsearch_bulk",
"size": 10000
"_op": "elasticsearch_bulk",
"size": 10000
}
]
}
}
20 changes: 17 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@
"private": true,
"scripts": {
"start": "node service.js",
"setup": "lerna bootstrap && lerna link --force-local && lerna run build",
"setup": "lerna bootstrap && lerna link --force-local && yarn build",
"bootstrap:prod": "lerna bootstrap -- --production=true --link-duplicates --no-optional",
"bootstrap": "lerna bootstrap",
"build": "lerna run build --parallel --prefix=false",
"build:prod": "lerna run build:prod --parallel --prefix=false",
"build:watch": "lerna run build:watch --parallel --stream --prefix",
"bump": "./scripts/bump.js",
"benchmark": "lerna run benchmark",
"test": "jest --ci --cache --silent --maxWorkers=4 --detectOpenHandles --forceExit",
"test:ci": "yarn test --bail=true",
"test:watch": "jest --coverage=false --notify --onlyChanged --watch",
"test:e2e": "yarn --cwd ./e2e test",
"test:e2e:ci": "yarn --cwd ./e2e test:ci",
Expand All @@ -21,15 +23,27 @@
"publish:changed": "./scripts/publish.sh"
},
"dependencies": {
"@types/bluebird": "^3.5.24",
"@types/bunyan": "^1.8.5",
"@types/convict": "^4.2.0",
"@types/debug": "^0.0.31",
"@types/fs-extra": "^5.0.4",
"@types/lodash": "^4.14.117",
"@types/lodash.clonedeep": "^4.5.4",
"@types/nanoid": "^1.2.0",
"@types/node": "^10.12.0",
"@types/p-event": "^1.3.0",
"@types/socket.io": "^1.4.38",
"@types/socket.io-client": "^1.4.32",
"@types/uuid": "^3.4.4",
"lerna": "^3.4.3",
"typescript": "^3.1.3"
},
"devDependencies": {
"@types/jest": "^23.3.5",
"@types/jest": "^23.3.8",
"babel-core": "^6.0.0",
"babel-jest": "^23.6.0",
"eslint": "^5.7.0",
"eslint": "^5.8.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"fs-extra": "^7.0.0",
Expand Down
2 changes: 1 addition & 1 deletion packages/docker-compose-js/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"debug": "^4.1.0"
},
"devDependencies": {
"eslint": "^5.7.0",
"eslint": "^5.8.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"jest": "^23.6.0",
Expand Down
2 changes: 1 addition & 1 deletion packages/elasticsearch-api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
},
"devDependencies": {
"debug": "^4.1.0",
"eslint": "^5.7.0",
"eslint": "^5.8.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"jest": "^23.6.0",
Expand Down
2 changes: 1 addition & 1 deletion packages/error-parser/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"lodash": "^4.17.11"
},
"devDependencies": {
"eslint": "^5.7.0",
"eslint": "^5.8.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"jest": "^23.6.0",
Expand Down
79 changes: 79 additions & 0 deletions packages/job-components/bench/data-entity-large-suite.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
'use strict';

const { Suite } = require('./helpers');
const FakeDataEntity = require('./fixtures/fake-data-entity');
const { DataEntity } = require('../dist');

const data = {};

for (let i = 0; i < 100; i++) {
data[`str-${i}`] = `data-${i}`;
data[`int-${i}`] = i;
}

const metadata = { id: Math.random() * 1000 * 1000 };

module.exports = () => Suite('DataEntity (large records)')
.add('new data', {
fn() {
let entity = Object.assign({}, data);
entity.metadata = Object.assign({ createdAt: Date.now() });
entity = null;
return entity;
}
})
.add('new data with metadata', {
fn() {
let entity = Object.assign({}, data);
entity.metadata = Object.assign({}, metadata, { createdAt: Date.now() });
entity = null;
return entity;
}
})
.add('new FakeDataEntity', {
fn() {
let entity = new FakeDataEntity(data);
entity = null;
return entity;
}
})
.add('new FakeDataEntity metadata', {
fn() {
let entity = new FakeDataEntity(data, metadata);
entity = null;
return entity;
}
})
.add('new DataEntity', {
fn() {
let entity = new DataEntity(data);
entity = null;
return entity;
}
})
.add('new DataEntity with metadata', {
fn() {
let entity = new DataEntity(data, metadata);
entity = null;
return entity;
}
})
.add('DataEntity.make', {
fn() {
let entity = DataEntity.make(data);
entity = null;
return entity;
}
})
.add('DataEntity.make with metadata', {
fn() {
let entity = DataEntity.make(data, metadata);
entity = null;
return entity;
}
})
.run({
async: true,
initCount: 2,
maxTime: 5,
});
Loading

0 comments on commit c39e906

Please sign in to comment.