v0.44.1 Fix a couple small issues in job-components and teraslice-tes…

…t-harness (#908)

* fix issue with npmified Logger definitions

* test clients create fn should return a { client: any } not any

* bump test harnesses and job-components

* fix type definition issues with runSlice and createSlices

* add support ignoring certain directories when loading typescript operations

* add failing reader spec

* fix teraslice-op-test-harness client usage

* when max_retries is set to zero, it shouldn't fire slice:retry

* bump teraslice to v0.44.1

package.json

@@ -24,7 +24,6 @@
     },
     "dependencies": {
         "@types/bluebird": "^3.5.24",
-        "@types/bunyan": "^1.8.5",
         "@types/convict": "^4.2.0",
         "@types/debug": "^0.0.31",
         "@types/fs-extra": "^5.0.4",

packages/job-components/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@terascope/job-components",
-    "version": "0.11.0",
+    "version": "0.11.1",
     "publishConfig": {
         "access": "public"
     },

packages/job-components/src/interfaces/context.ts

@@ -1,16 +1,7 @@
-// @ts-ignore
-import bunyan from '@types/bunyan';
-import Stream from 'stream';
 import { EventEmitter } from 'events';
 import { OpConfig } from './jobs';
 import { ExecutionContextAPI } from '../execution-context';
-
-export type LoggerStream = Stream|WritableStream|undefined;
-
-export interface Logger extends bunyan {
-    streams: LoggerStream[];
-    flush(): Promise<void>;
-}
+import { Logger } from './logger';
 
 export interface ClusterStateConfig {
     connection: string|'default';
@@ -75,7 +66,7 @@ export interface ConnectionConfig {
     type: string;
 }
 
-export type ClientFactoryFn = (config: object, logger: Logger, options: ConnectionConfig) => any;
+export type ClientFactoryFn = (config: object, logger: Logger, options: ConnectionConfig) => { client: any };
 
 export interface FoundationApis {
     makeLogger(...params: any[]): Logger;

packages/job-components/src/interfaces/index.ts

@@ -1,4 +1,5 @@
 export * from './context';
 export * from './jobs';
+export * from './logger';
 export * from './operation-lifecycle';
 export * from './operations';

packages/job-components/src/interfaces/logger.ts

@@ -0,0 +1,308 @@
+import Stream from 'stream';
+import { EventEmitter } from 'events';
+
+// This includes additional properties from terafoundation
+// Type definitions for bunyan 1.8
+// Project: https://github.com/trentm/node-bunyan
+// Definitions by: Alex Mikhalev <https://github.com/amikhalev>
+// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
+// TypeScript Version: 2.1
+
+declare class Logger extends EventEmitter {
+    constructor(options: Logger.LoggerOptions);
+    addStream(stream: Logger.Stream): void;
+    addSerializers(serializers: Logger.Serializers): void;
+    child(options: Object, simple?: boolean): Logger;
+    reopenFileStreams(): void;
+
+    level(): number;
+    level(value: Logger.LogLevel): void;
+    levels(): number[];
+    levels(name: number | string): number;
+    levels(name: number | string, value: Logger.LogLevel): void;
+
+    /**
+     * Terafoundation specific
+    */
+    streams: Stream|WritableStream|undefined[];
+
+    fields: any;
+    src: boolean;
+
+    /**
+     * Returns a boolean: is the `trace` level enabled?
+     *
+     * This is equivalent to `log.isTraceEnabled()` or `log.isEnabledFor(TRACE)` in log4j.
+     */
+    trace(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    trace(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    trace(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    trace(format: any, ...params: any[]): void;
+
+    /**
+     * Returns a boolean: is the `debug` level enabled?
+     *
+     * This is equivalent to `log.isDebugEnabled()` or `log.isEnabledFor(DEBUG)` in log4j.
+     */
+    debug(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    debug(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    debug(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    debug(format: any, ...params: any[]): void;
+
+    /**
+     * Returns a boolean: is the `info` level enabled?
+     *
+     * This is equivalent to `log.isInfoEnabled()` or `log.isEnabledFor(INFO)` in log4j.
+     */
+    info(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    info(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    info(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    info(format: any, ...params: any[]): void;
+
+    /**
+     * Returns a boolean: is the `warn` level enabled?
+     *
+     * This is equivalent to `log.isWarnEnabled()` or `log.isEnabledFor(WARN)` in log4j.
+     */
+    warn(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    warn(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    warn(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    warn(format: any, ...params: any[]): void;
+
+    /**
+     * Returns a boolean: is the `error` level enabled?
+     *
+     * This is equivalent to `log.isErrorEnabled()` or `log.isEnabledFor(ERROR)` in log4j.
+     */
+    error(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    error(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    error(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    error(format: any, ...params: any[]): void;
+
+    /**
+     * Returns a boolean: is the `fatal` level enabled?
+     *
+     * This is equivalent to `log.isFatalEnabled()` or `log.isEnabledFor(FATAL)` in log4j.
+     */
+    fatal(): boolean;
+
+    /**
+     * Special case to log an `Error` instance to the record.
+     * This adds an `err` field with exception details
+     * (including the stack) and sets `msg` to the exception
+     * message or you can specify the `msg`.
+     */
+    fatal(error: Error, ...params: any[]): void;
+
+    /**
+     * The first field can optionally be a "fields" object, which
+     * is merged into the log record.
+     *
+     * To pass in an Error *and* other fields, use the `err`
+     * field name for the Error instance.
+     */
+    fatal(obj: Object, ...params: any[]): void;
+
+    /**
+     * Uses `util.format` for msg formatting.
+     */
+    fatal(format: any, ...params: any[]): void;
+
+    /**
+     * Terafoundation specific, flush the logs
+    */
+    flush(): Promise<void>;
+}
+
+declare namespace Logger {
+    const TRACE: number;
+    const DEBUG: number;
+    const INFO: number;
+    const WARN: number;
+    const ERROR: number;
+    const FATAL: number;
+
+    type LogLevelString = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
+    type LogLevel = LogLevelString | number;
+
+    const levelFromName: { [name in LogLevelString]: number };
+    const nameFromLevel: { [level: number]: string };
+
+    const stdSerializers: StdSerializers;
+
+    function createLogger(options: LoggerOptions): Logger;
+
+    function safeCycles(): (key: string, value: any) => any;
+
+    function resolveLevel(value: LogLevel): number;
+
+    interface Stream {
+        type?: string;
+        level?: LogLevel;
+        path?: string;
+        stream?: NodeJS.WritableStream | Stream;
+        closeOnExit?: boolean;
+        period?: string;
+        count?: number;
+        name?: string;
+        reemitErrorEvents?: boolean;
+    }
+
+    interface LoggerOptions {
+        name: string;
+        streams?: Stream[];
+        level?: LogLevel;
+        stream?: NodeJS.WritableStream;
+        serializers?: Serializers;
+        src?: boolean;
+        [custom: string]: any;
+    }
+
+    type Serializer = (input: any) => any;
+
+    interface Serializers {
+        [key: string]: Serializer;
+    }
+
+    interface StdSerializers extends Serializers {
+        err: Serializer;
+        res: Serializer;
+        req: Serializer;
+    }
+
+    interface RingBufferOptions {
+        limit?: number;
+    }
+
+    class RingBuffer extends EventEmitter implements NodeJS.WritableStream {
+        constructor(options: RingBufferOptions);
+
+        writable: boolean;
+        records: any[];
+
+        write(record: any): boolean;
+        end(record?: any): void;
+        destroy(): void;
+        destroySoon(): void;
+    }
+
+    interface RotatingFileStreamOptions {
+        path: string;
+        count?: number;
+        period?: string;
+    }
+
+    class RotatingFileStream extends EventEmitter implements NodeJS.WritableStream {
+        constructor(options: RotatingFileStreamOptions);
+
+        writable: boolean;
+        periodNum: number;
+        periodScope: string;
+        stream: any;
+        rotQueue: any[];
+        rotating: boolean;
+
+        write(record: any): boolean;
+        end(record?: any): void;
+        destroy(): void;
+        destroySoon(): void;
+        rotate(): void;
+    }
+}
+
+export { Logger };

packages/job-components/src/interfaces/operations.ts

@@ -1,6 +1,7 @@
 import { Schema } from 'convict';
 import { ValidatedJobConfig, OpConfig, ExecutionConfig, LegacyExecutionContext } from './jobs';
-import { Context, SysConfig, Logger } from './context';
+import { Context, SysConfig } from './context';
+import { Logger } from './logger';
 
 export type crossValidationFn = (job: ValidatedJobConfig, sysconfig: SysConfig) => void;
 export type selfValidationFn = (config: OpConfig) => void;

packages/job-components/src/operation-loader.ts

@@ -310,7 +310,8 @@ export class OperationLoader {
         const allowedNames = uniq([name, ...codeNames]);
 
         const invalid = [
-            'node_modules'
+            'node_modules',
+            ...ignoreDirectories(),
         ];
 
         const findCode = (rootDir: string): string|null => {
@@ -355,3 +356,9 @@ function availableExtensions(): string[] {
     // @ts-ignore
     return global.availableExtensions ? global.availableExtensions : ['.js'];
 }
+
+function ignoreDirectories() {
+    // populated by teraslice Jest configuration
+    // @ts-ignore
+    return global.ignoreDirectories || [];
+}