diff --git a/.changeset/giant-elephants-drop.md b/.changeset/giant-elephants-drop.md new file mode 100644 index 0000000000..2403d56383 --- /dev/null +++ b/.changeset/giant-elephants-drop.md @@ -0,0 +1,5 @@ +--- +'@chainlink/coinpaprika-adapter': minor +--- + +update rate limits for coinpaprika diff --git a/packages/core/bootstrap/src/index.ts b/packages/core/bootstrap/src/index.ts index 6efdcab7f6..d8ff1cdd23 100644 --- a/packages/core/bootstrap/src/index.ts +++ b/packages/core/bootstrap/src/index.ts @@ -1,7 +1,5 @@ import { - AdapterMetricsMeta, AdapterRequest, - AdapterErrorLog, AdapterContext, Execute, ExecuteSync, @@ -12,24 +10,20 @@ import { Config, } from '@chainlink/types' import { combineReducers, Store } from 'redux' -import { Cache, withCache } from './lib/cache' -import * as cacheWarmer from './lib/cache-warmer' -import { WARMUP_REQUEST_ID } from './lib/cache-warmer/config' -import { - AdapterError, - logger as Logger, - Requester, - Validator, - Builder, - normalizeInput, -} from './lib/external-adapter' +import { Cache, withCache } from './lib/middleware/cache' +import * as cacheWarmer from './lib/middleware/cache-warmer' +import { AdapterError, logger as Logger, Requester, Validator, Builder } from './lib/modules' import * as metrics from './lib/metrics' -import * as RateLimit from './lib/rate-limit' -import * as burstLimit from './lib/burst-limit' +import * as RateLimit from './lib/middleware/rate-limit' +import * as burstLimit from './lib/middleware/burst-limit' +import * as ioLogger from './lib/middleware/io-logger' +import * as statusCode from './lib/middleware/status-code' +import * as debug from './lib/middleware/debugger' +import * as normalize from './lib/middleware/normalize' import * as server from './lib/server' import { configureStore } from './lib/store' import * as util from './lib/util' -import * as ws from './lib/ws' +import * as ws from './lib/middleware/ws' import http from 'http' const REDUX_MIDDLEWARE = ['burstLimit', 'cacheWarmer', 'rateLimit', 'ws'] as const @@ -61,122 +55,6 @@ export const storeSlice = (slice: ReduxMiddleware): Store => dispatch: (a) => store.dispatch(a), } as Store) -// Make sure data has the same statusCode as the one we got as a result -const withStatusCode: Middleware = async (execute, context) => async (input) => { - const { statusCode, data, ...rest } = await execute(input, context) - if (data && typeof data === 'object' && data.statusCode) { - return { - ...rest, - statusCode, - data: { - ...data, - statusCode, - }, - } - } - - return { ...rest, statusCode, data } -} - -// Log adapter input & output data -const withLogger: Middleware = async (execute, context) => async (input: AdapterRequest) => { - Logger.debug('Input: ', { input }) - try { - const result = await execute(input, context) - Logger.debug(`Output: [${result.statusCode}]: `, { output: result.data }) - return result - } catch (error) { - const feedID = metrics.util.getFeedId(input) - const errorLog: AdapterErrorLog = { - message: error.toString(), - jobRunID: input.id, - params: input.data, - feedID, - url: error.url, - errorResponse: error.errorResponse, - } - - if (Logger.level === 'debug') { - errorLog.stack = error.stack - } - - if (Logger.level === 'trace') { - errorLog.rawError = error.cause - errorLog.stack = undefined - } - - Logger.error(errorLog) - throw error - } -} - -const withMetrics: Middleware = async (execute, context) => async (input: AdapterRequest) => { - const feedId = metrics.util.getFeedId(input) - const metricsMeta: AdapterMetricsMeta = { - feedId, - } - - const recordMetrics = () => { - const labels: Parameters[0] = { - is_cache_warming: String(input.id === WARMUP_REQUEST_ID), - method: 'POST', - feed_id: feedId, - } - const end = metrics.httpRequestDurationSeconds.startTimer() - - return (props: { - providerStatusCode?: number - statusCode?: number - type?: metrics.HttpRequestType - }) => { - labels.type = props.type - labels.status_code = metrics.util.normalizeStatusCode(props.statusCode) - labels.provider_status_code = metrics.util.normalizeStatusCode(props.providerStatusCode) - end() - metrics.httpRequestsTotal.labels(labels).inc() - } - } - - const record = recordMetrics() - try { - const result = await execute({ ...input, metricsMeta }, context) - record({ - statusCode: result.statusCode, - type: - result.data.maxAge || (result as any).maxAge - ? metrics.HttpRequestType.CACHE_HIT - : metrics.HttpRequestType.DATA_PROVIDER_HIT, - }) - return { ...result, metricsMeta: { ...result.metricsMeta, ...metricsMeta } } - } catch (error) { - const providerStatusCode: number | undefined = error.cause?.response?.status - record({ - statusCode: providerStatusCode ? 200 : 500, - providerStatusCode, - type: providerStatusCode - ? metrics.HttpRequestType.DATA_PROVIDER_HIT - : metrics.HttpRequestType.ADAPTER_ERROR, - }) - throw error - } -} - -export const withDebug: Middleware = async (execute, context) => async (input: AdapterRequest) => { - const result = await execute(input, context) - if (!util.isDebug()) { - const { debug, ...rest } = result - return rest - } - return result -} - -export const withNormalizedInput: ( - endpointSelector?: (request: AdapterRequest) => APIEndpoint, -) => Middleware = (endpointSelector) => async (execute, context) => async (input: AdapterRequest) => { - const normalizedInput = endpointSelector ? normalizeInput(input, endpointSelector(input)) : input - return execute(normalizedInput, context) -} - export const makeMiddleware = ( execute: Execute, makeWsHandler?: MakeWSHandler, @@ -185,12 +63,12 @@ export const makeMiddleware = ( const warmerMiddleware = [ withCache(storeSlice('burstLimit')), RateLimit.withRateLimit(storeSlice('rateLimit')), - withStatusCode, - withNormalizedInput(endpointSelector), - ].concat(metrics.METRICS_ENABLED ? [withMetrics] : []) + statusCode.withStatusCode, + normalize.withNormalizedInput(endpointSelector), + ].concat(metrics.METRICS_ENABLED ? [metrics.withMetrics] : []) return [ - withLogger, + ioLogger.withIOLogger, withCache(storeSlice('burstLimit')), cacheWarmer.withCacheWarmer(storeSlice('cacheWarmer'), warmerMiddleware, { store: storeSlice('ws'), @@ -198,9 +76,9 @@ export const makeMiddleware = ( })(execute), ws.withWebSockets(storeSlice('ws'), makeWsHandler), RateLimit.withRateLimit(storeSlice('rateLimit')), - withStatusCode, - withNormalizedInput(endpointSelector), - ].concat(metrics.METRICS_ENABLED ? [withMetrics, withDebug] : [withDebug]) + statusCode.withStatusCode, + normalize.withNormalizedInput(endpointSelector), + ].concat(metrics.METRICS_ENABLED ? [metrics.withMetrics, debug.withDebug] : [debug.withDebug]) } // Wrap raw Execute function with middleware diff --git a/packages/core/bootstrap/src/lib/external-adapter/config.ts b/packages/core/bootstrap/src/lib/config/index.ts similarity index 97% rename from packages/core/bootstrap/src/lib/external-adapter/config.ts rename to packages/core/bootstrap/src/lib/config/index.ts index 75c3e642bc..6661b62423 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/config.ts +++ b/packages/core/bootstrap/src/lib/config/index.ts @@ -1,6 +1,6 @@ import { getRandomRequiredEnv, getRandomEnv, getEnv } from '../util' import { Config } from '@chainlink/types' -import { logger } from './logger' +import { logger } from '../modules/logger' const ENV_API_KEY = 'API_KEY' const ENV_API_ENDPOINT = 'API_ENDPOINT' diff --git a/packages/core/bootstrap/src/lib/external-adapter/overrides/presetIncludes.json b/packages/core/bootstrap/src/lib/config/overrides/presetIncludes.json similarity index 100% rename from packages/core/bootstrap/src/lib/external-adapter/overrides/presetIncludes.json rename to packages/core/bootstrap/src/lib/config/overrides/presetIncludes.json diff --git a/packages/core/bootstrap/src/lib/external-adapter/overrides/presetSymbols.json b/packages/core/bootstrap/src/lib/config/overrides/presetSymbols.json similarity index 100% rename from packages/core/bootstrap/src/lib/external-adapter/overrides/presetSymbols.json rename to packages/core/bootstrap/src/lib/config/overrides/presetSymbols.json diff --git a/packages/core/bootstrap/src/lib/external-adapter/overrides/presetTokens.json b/packages/core/bootstrap/src/lib/config/overrides/presetTokens.json similarity index 100% rename from packages/core/bootstrap/src/lib/external-adapter/overrides/presetTokens.json rename to packages/core/bootstrap/src/lib/config/overrides/presetTokens.json diff --git a/packages/core/bootstrap/src/lib/provider-limits/index.ts b/packages/core/bootstrap/src/lib/config/provider-limits/index.ts similarity index 98% rename from packages/core/bootstrap/src/lib/provider-limits/index.ts rename to packages/core/bootstrap/src/lib/config/provider-limits/index.ts index 5a53efbbaa..ab637a249a 100644 --- a/packages/core/bootstrap/src/lib/provider-limits/index.ts +++ b/packages/core/bootstrap/src/lib/config/provider-limits/index.ts @@ -1,4 +1,4 @@ -import { logger } from '../external-adapter' +import { logger } from '../../modules' import limits from './limits.json' export const DEFAULT_MINUTE_RATE_LIMIT = 60 diff --git a/packages/core/bootstrap/src/lib/provider-limits/limits.json b/packages/core/bootstrap/src/lib/config/provider-limits/limits.json similarity index 98% rename from packages/core/bootstrap/src/lib/provider-limits/limits.json rename to packages/core/bootstrap/src/lib/config/provider-limits/limits.json index 06be772082..92bda899b1 100644 --- a/packages/core/bootstrap/src/lib/provider-limits/limits.json +++ b/packages/core/bootstrap/src/lib/config/provider-limits/limits.json @@ -379,9 +379,22 @@ "http": { "free": { "rateLimit1s": 10, - "rateLimit1m": 600, - "rateLimit1h": 34246.5, - "note": "600/m and 25M/mo for free" + "rateLimit1h": 69.44, + "note": "50k/mo for free" + }, + "pro": { + "rateLimit1s": 10, + "rateLimit1h": 347.22, + "note": "250k/mo for pro" + }, + "business": { + "rateLimit1s": 10, + "rateLimit1h": 1388.888, + "note": "1mil/mo for business" + }, + "enterprise": { + "rateLimit1s": 10, + "note": "unlimited per month" } }, "ws": {} diff --git a/packages/core/bootstrap/src/lib/config/test-payload-loader.ts b/packages/core/bootstrap/src/lib/config/test-payload-loader.ts index 64c57d0d58..002f0b1caf 100644 --- a/packages/core/bootstrap/src/lib/config/test-payload-loader.ts +++ b/packages/core/bootstrap/src/lib/config/test-payload-loader.ts @@ -1,5 +1,5 @@ import Ajv, { JSONSchemaType } from 'ajv' -import { logger } from '../external-adapter' +import { logger } from '../modules' import path from 'path' import { AdapterRequestData } from '@chainlink/types' diff --git a/packages/core/bootstrap/src/lib/metrics/index.ts b/packages/core/bootstrap/src/lib/metrics/index.ts index 8e99291c76..70480dbef7 100644 --- a/packages/core/bootstrap/src/lib/metrics/index.ts +++ b/packages/core/bootstrap/src/lib/metrics/index.ts @@ -1,6 +1,8 @@ import * as client from 'prom-client' import { parseBool } from '../util' -export * as util from './util' +import { WARMUP_REQUEST_ID } from '../middleware/cache-warmer/config' +import * as util from './util' +import { Middleware, AdapterRequest, AdapterMetricsMeta } from '@chainlink/types' export const setupMetrics = (name: string): void => { client.collectDefaultMetrics() @@ -12,6 +14,58 @@ export const setupMetrics = (name: string): void => { export const METRICS_ENABLED = parseBool(process.env.EXPERIMENTAL_METRICS_ENABLED) +export const withMetrics: Middleware = + async (execute, context) => async (input: AdapterRequest) => { + const feedId = util.getFeedId(input) + const metricsMeta: AdapterMetricsMeta = { + feedId, + } + + const recordMetrics = () => { + const labels: Parameters[0] = { + is_cache_warming: String(input.id === WARMUP_REQUEST_ID), + method: 'POST', + feed_id: feedId, + } + const end = httpRequestDurationSeconds.startTimer() + + return (props: { + providerStatusCode?: number + statusCode?: number + type?: HttpRequestType + }) => { + labels.type = props.type + labels.status_code = util.normalizeStatusCode(props.statusCode) + labels.provider_status_code = util.normalizeStatusCode(props.providerStatusCode) + end() + httpRequestsTotal.labels(labels).inc() + } + } + + const record = recordMetrics() + try { + const result = await execute({ ...input, metricsMeta }, context) + record({ + statusCode: result.statusCode, + type: + result.data.maxAge || (result as any).maxAge + ? HttpRequestType.CACHE_HIT + : HttpRequestType.DATA_PROVIDER_HIT, + }) + return { ...result, metricsMeta: { ...result.metricsMeta, ...metricsMeta } } + } catch (error) { + const providerStatusCode: number | undefined = error.cause?.response?.status + record({ + statusCode: providerStatusCode ? 200 : 500, + providerStatusCode, + type: providerStatusCode + ? HttpRequestType.DATA_PROVIDER_HIT + : HttpRequestType.ADAPTER_ERROR, + }) + throw error + } + } + export enum HttpRequestType { CACHE_HIT = 'cacheHit', DATA_PROVIDER_HIT = 'dataProviderHit', @@ -62,3 +116,5 @@ export const httpRateLimit = new client.Counter({ name: 'http_requests_rate_limit', help: 'The number of denied requests because of server rate limiting', }) + +export * as util from './util' diff --git a/packages/core/bootstrap/src/lib/metrics/util.ts b/packages/core/bootstrap/src/lib/metrics/util.ts index 5ea5ebdc7b..5fb31e75ce 100644 --- a/packages/core/bootstrap/src/lib/metrics/util.ts +++ b/packages/core/bootstrap/src/lib/metrics/util.ts @@ -1,5 +1,5 @@ import { AdapterRequest } from '@chainlink/types' -import { logger, Validator } from '../external-adapter' +import { logger, Validator } from '../modules' import { excludableAdapterRequestProperties } from '../util' import * as crypto from 'crypto' diff --git a/packages/core/bootstrap/src/lib/burst-limit/actions.ts b/packages/core/bootstrap/src/lib/middleware/burst-limit/actions.ts similarity index 100% rename from packages/core/bootstrap/src/lib/burst-limit/actions.ts rename to packages/core/bootstrap/src/lib/middleware/burst-limit/actions.ts diff --git a/packages/core/bootstrap/src/lib/burst-limit/index.ts b/packages/core/bootstrap/src/lib/middleware/burst-limit/index.ts similarity index 98% rename from packages/core/bootstrap/src/lib/burst-limit/index.ts rename to packages/core/bootstrap/src/lib/middleware/burst-limit/index.ts index 4ea2a8041c..974cd884a9 100644 --- a/packages/core/bootstrap/src/lib/burst-limit/index.ts +++ b/packages/core/bootstrap/src/lib/middleware/burst-limit/index.ts @@ -8,7 +8,7 @@ import { } from './reducer' import * as actions from './actions' import { WARMUP_BATCH_REQUEST_ID } from '../cache-warmer/config' -import { AdapterError, logger } from '../external-adapter' +import { AdapterError, logger } from '../../modules' export * as actions from './actions' export * as reducer from './reducer' diff --git a/packages/core/bootstrap/src/lib/burst-limit/reducer.ts b/packages/core/bootstrap/src/lib/middleware/burst-limit/reducer.ts similarity index 100% rename from packages/core/bootstrap/src/lib/burst-limit/reducer.ts rename to packages/core/bootstrap/src/lib/middleware/burst-limit/reducer.ts diff --git a/packages/core/bootstrap/src/lib/cache-warmer/actions.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/actions.ts similarity index 100% rename from packages/core/bootstrap/src/lib/cache-warmer/actions.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/actions.ts diff --git a/packages/core/bootstrap/src/lib/cache-warmer/config.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/config.ts similarity index 94% rename from packages/core/bootstrap/src/lib/cache-warmer/config.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/config.ts index d073ea13f8..e0b63f24dd 100644 --- a/packages/core/bootstrap/src/lib/cache-warmer/config.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache-warmer/config.ts @@ -1,6 +1,6 @@ import objectHash from 'object-hash' -import { getHashOpts } from '../util' -import { logger } from '../external-adapter' +import { getHashOpts } from '../../util' +import { logger } from '../../modules' export const WARMUP_REQUEST_ID = '9001' export const WARMUP_BATCH_REQUEST_ID = '9002' diff --git a/packages/core/bootstrap/src/lib/cache-warmer/epics.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/epics.ts similarity index 99% rename from packages/core/bootstrap/src/lib/cache-warmer/epics.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/epics.ts index 5d458aa68e..0aed98b0cc 100644 --- a/packages/core/bootstrap/src/lib/cache-warmer/epics.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache-warmer/epics.ts @@ -15,7 +15,7 @@ import { takeUntil, withLatestFrom, } from 'rxjs/operators' -import { RootState } from '../..' +import { RootState } from '../../..' import { warmupExecute, warmupFailed, @@ -43,7 +43,7 @@ import { import { concatenateBatchResults, getSubscriptionKey, splitIntoBatches } from './util' import { getTTL, getMaxAgeOverride } from '../cache/ttl' import * as metrics from './metrics' -import { getFeedId } from '../metrics/util' +import { getFeedId } from '../../metrics/util' import { PayloadAction } from '@reduxjs/toolkit' export interface EpicDependencies { diff --git a/packages/core/bootstrap/src/lib/cache-warmer/index.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/index.ts similarity index 96% rename from packages/core/bootstrap/src/lib/cache-warmer/index.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/index.ts index 5411faeb75..1682187a65 100644 --- a/packages/core/bootstrap/src/lib/cache-warmer/index.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache-warmer/index.ts @@ -1,9 +1,9 @@ import { AdapterRequest, Execute, MakeWSHandler, Middleware } from '@chainlink/types' import { Store } from 'redux' -import { withMiddleware } from '../../index' -import { logger } from '../external-adapter' -import { getFeedId } from '../metrics/util' -import * as util from '../util' +import { withMiddleware } from '../../../index' +import { logger } from '../../modules' +import { getFeedId } from '../../metrics/util' +import * as util from '../../util' import { getWSConfig } from '../ws/config' import { getSubsId, RootState as WSState } from '../ws/reducer' import { separateBatches } from '../ws/utils' diff --git a/packages/core/bootstrap/src/lib/cache-warmer/metrics.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/metrics.ts similarity index 100% rename from packages/core/bootstrap/src/lib/cache-warmer/metrics.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/metrics.ts diff --git a/packages/core/bootstrap/src/lib/cache-warmer/reducer.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/reducer.ts similarity index 99% rename from packages/core/bootstrap/src/lib/cache-warmer/reducer.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/reducer.ts index 350cac99fe..eee9731748 100644 --- a/packages/core/bootstrap/src/lib/cache-warmer/reducer.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache-warmer/reducer.ts @@ -1,6 +1,6 @@ import { AdapterRequest, Execute } from '@chainlink/types' import { combineReducers, createReducer } from '@reduxjs/toolkit' -import { logger } from '../external-adapter' +import { logger } from '../../modules' import * as actions from './actions' import { getSubscriptionKey } from './util' import { merge, uniq } from 'lodash' diff --git a/packages/core/bootstrap/src/lib/cache-warmer/util.ts b/packages/core/bootstrap/src/lib/middleware/cache-warmer/util.ts similarity index 98% rename from packages/core/bootstrap/src/lib/cache-warmer/util.ts rename to packages/core/bootstrap/src/lib/middleware/cache-warmer/util.ts index d34876cc5b..f239fa519b 100644 --- a/packages/core/bootstrap/src/lib/cache-warmer/util.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache-warmer/util.ts @@ -3,7 +3,7 @@ import { WarmupExecutePayload, WarmupSubscribedPayload } from './actions' import { get } from './config' import { BatchableProperty, SubscriptionData } from './reducer' import { AdapterRequest, AdapterResponse } from '@chainlink/types' -import { hash } from '../util' +import { hash } from '../../util' const conf = get() export function getSubscriptionKey( diff --git a/packages/core/bootstrap/src/lib/cache/index.ts b/packages/core/bootstrap/src/lib/middleware/cache/index.ts similarity index 99% rename from packages/core/bootstrap/src/lib/cache/index.ts rename to packages/core/bootstrap/src/lib/middleware/cache/index.ts index 5d536e456e..470d3e0fd6 100644 --- a/packages/core/bootstrap/src/lib/cache/index.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache/index.ts @@ -5,7 +5,7 @@ import { Execute, Middleware, } from '@chainlink/types' -import { logger } from '../external-adapter' +import { logger } from '../../modules' import { Store } from 'redux' import { reducer } from '../burst-limit' import { withBurstLimit } from '../burst-limit' @@ -17,7 +17,7 @@ import { parseBool, uuid, hash, -} from '../util' +} from '../../util' import { getMaxAgeOverride, getTTL } from './ttl' import * as local from './local' import { LocalOptions } from './local' diff --git a/packages/core/bootstrap/src/lib/cache/local.ts b/packages/core/bootstrap/src/lib/middleware/cache/local.ts similarity index 98% rename from packages/core/bootstrap/src/lib/cache/local.ts rename to packages/core/bootstrap/src/lib/middleware/cache/local.ts index 1a6f0b3a29..dd376fdfc5 100644 --- a/packages/core/bootstrap/src/lib/cache/local.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache/local.ts @@ -1,5 +1,5 @@ import LRU from 'lru-cache' -import { parseBool } from '../util' +import { parseBool } from '../../util' import { CacheEntry } from './types' // Options diff --git a/packages/core/bootstrap/src/lib/cache/metrics.ts b/packages/core/bootstrap/src/lib/middleware/cache/metrics.ts similarity index 98% rename from packages/core/bootstrap/src/lib/cache/metrics.ts rename to packages/core/bootstrap/src/lib/middleware/cache/metrics.ts index 2ee523f235..f654eeb021 100644 --- a/packages/core/bootstrap/src/lib/cache/metrics.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache/metrics.ts @@ -1,5 +1,5 @@ import * as client from 'prom-client' -import { normalizeStatusCode } from '../metrics/util' +import { normalizeStatusCode } from '../../metrics/util' interface CacheExecutionDurationParams { participantId: string diff --git a/packages/core/bootstrap/src/lib/cache/redis/index.ts b/packages/core/bootstrap/src/lib/middleware/cache/redis/index.ts similarity index 100% rename from packages/core/bootstrap/src/lib/cache/redis/index.ts rename to packages/core/bootstrap/src/lib/middleware/cache/redis/index.ts diff --git a/packages/core/bootstrap/src/lib/cache/redis/metrics.ts b/packages/core/bootstrap/src/lib/middleware/cache/redis/metrics.ts similarity index 100% rename from packages/core/bootstrap/src/lib/cache/redis/metrics.ts rename to packages/core/bootstrap/src/lib/middleware/cache/redis/metrics.ts diff --git a/packages/core/bootstrap/src/lib/cache/redis/redis.ts b/packages/core/bootstrap/src/lib/middleware/cache/redis/redis.ts similarity index 99% rename from packages/core/bootstrap/src/lib/cache/redis/redis.ts rename to packages/core/bootstrap/src/lib/middleware/cache/redis/redis.ts index e60c1e6cba..01d25e822a 100644 --- a/packages/core/bootstrap/src/lib/cache/redis/redis.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache/redis/redis.ts @@ -3,7 +3,7 @@ import { createClient } from 'redis' // TODO https://app.shortcut.com/chainlinklabs/story/23811/update-redis-client-types-and-imports import { RedisClientOptions } from '@node-redis/client/dist/lib/client' //TODO add RedisClientType here import { RedisModules, RedisScripts } from '@node-redis/client/dist/lib/commands' -import { logger } from '../../external-adapter' +import { logger } from '../../../modules' import { CacheEntry } from '../types' import * as metrics from './metrics' diff --git a/packages/core/bootstrap/src/lib/cache/ttl.ts b/packages/core/bootstrap/src/lib/middleware/cache/ttl.ts similarity index 97% rename from packages/core/bootstrap/src/lib/cache/ttl.ts rename to packages/core/bootstrap/src/lib/middleware/cache/ttl.ts index a033839ba9..990eef9361 100644 --- a/packages/core/bootstrap/src/lib/cache/ttl.ts +++ b/packages/core/bootstrap/src/lib/middleware/cache/ttl.ts @@ -1,4 +1,4 @@ -import { logger } from '../external-adapter' +import { logger } from '../../modules' import { AdapterRequest } from '@chainlink/types' import { CacheOptions, defaultOptions } from '.' diff --git a/packages/core/bootstrap/src/lib/cache/types.ts b/packages/core/bootstrap/src/lib/middleware/cache/types.ts similarity index 100% rename from packages/core/bootstrap/src/lib/cache/types.ts rename to packages/core/bootstrap/src/lib/middleware/cache/types.ts diff --git a/packages/core/bootstrap/src/lib/middleware/debugger/index.ts b/packages/core/bootstrap/src/lib/middleware/debugger/index.ts new file mode 100644 index 0000000000..a3391f0a0c --- /dev/null +++ b/packages/core/bootstrap/src/lib/middleware/debugger/index.ts @@ -0,0 +1,11 @@ +import { Middleware } from '@chainlink/types' +import { isDebug } from '../../util' + +export const withDebug: Middleware = async (execute, context) => async (input) => { + const result = await execute(input, context) + if (!isDebug()) { + const { debug, ...rest } = result + return rest + } + return result +} diff --git a/packages/core/bootstrap/src/lib/middleware/io-logger/index.ts b/packages/core/bootstrap/src/lib/middleware/io-logger/index.ts new file mode 100644 index 0000000000..0d7a16ae88 --- /dev/null +++ b/packages/core/bootstrap/src/lib/middleware/io-logger/index.ts @@ -0,0 +1,35 @@ +import type { Middleware, AdapterRequest, AdapterErrorLog } from '@chainlink/types' +import { logger as Logger } from '../../modules' +import { getFeedId } from '../../metrics/util' + +export const withIOLogger: Middleware = + async (execute, context) => async (input: AdapterRequest) => { + Logger.debug('Input: ', { input }) + try { + const result = await execute(input, context) + Logger.debug(`Output: [${result.statusCode}]: `, { output: result.data }) + return result + } catch (error) { + const feedID = getFeedId(input) + const errorLog: AdapterErrorLog = { + message: error.toString(), + jobRunID: input.id, + params: input.data, + feedID, + url: error.url, + errorResponse: error.errorResponse, + } + + if (Logger.level === 'debug') { + errorLog.stack = error.stack + } + + if (Logger.level === 'trace') { + errorLog.rawError = error.cause + errorLog.stack = undefined + } + + Logger.error(errorLog) + throw error + } + } diff --git a/packages/core/bootstrap/src/lib/middleware/normalize/index.ts b/packages/core/bootstrap/src/lib/middleware/normalize/index.ts new file mode 100644 index 0000000000..c2acc3dd0b --- /dev/null +++ b/packages/core/bootstrap/src/lib/middleware/normalize/index.ts @@ -0,0 +1,9 @@ +import type { Middleware, AdapterRequest, Config, APIEndpoint } from '@chainlink/types' +import { normalizeInput } from '../../modules' + +export const withNormalizedInput: ( + endpointSelector?: (request: AdapterRequest) => APIEndpoint, +) => Middleware = (endpointSelector) => async (execute, context) => async (input: AdapterRequest) => { + const normalizedInput = endpointSelector ? normalizeInput(input, endpointSelector(input)) : input + return execute(normalizedInput, context) +} diff --git a/packages/core/bootstrap/src/lib/rate-limit/actions.ts b/packages/core/bootstrap/src/lib/middleware/rate-limit/actions.ts similarity index 89% rename from packages/core/bootstrap/src/lib/rate-limit/actions.ts rename to packages/core/bootstrap/src/lib/middleware/rate-limit/actions.ts index 0819945508..9a552a64b1 100644 --- a/packages/core/bootstrap/src/lib/rate-limit/actions.ts +++ b/packages/core/bootstrap/src/lib/middleware/rate-limit/actions.ts @@ -1,6 +1,6 @@ import { AdapterRequest, AdapterResponse } from '@chainlink/types' import { createAction } from '@reduxjs/toolkit' -import { ActionBase, toActionPayload } from '../store' +import { ActionBase, toActionPayload } from '../../store' export interface SuccessfulRequestObservedPayload extends ActionBase { input: AdapterRequest diff --git a/packages/core/bootstrap/src/lib/rate-limit/config.ts b/packages/core/bootstrap/src/lib/middleware/rate-limit/config.ts similarity index 93% rename from packages/core/bootstrap/src/lib/rate-limit/config.ts rename to packages/core/bootstrap/src/lib/middleware/rate-limit/config.ts index feda7003b0..b34af7357f 100644 --- a/packages/core/bootstrap/src/lib/rate-limit/config.ts +++ b/packages/core/bootstrap/src/lib/middleware/rate-limit/config.ts @@ -1,6 +1,6 @@ -import { getRateLimit, getHTTPLimit } from '../provider-limits' -import { getEnv, parseBool } from '../util' -import { logger } from '../external-adapter' +import { getRateLimit, getHTTPLimit } from '../../config/provider-limits' +import { getEnv, parseBool } from '../../util' +import { logger } from '../../modules' import { AdapterContext } from '@chainlink/types' import { DEFAULT_CACHE_ENABLED } from '../cache' diff --git a/packages/core/bootstrap/src/lib/rate-limit/index.ts b/packages/core/bootstrap/src/lib/middleware/rate-limit/index.ts similarity index 98% rename from packages/core/bootstrap/src/lib/rate-limit/index.ts rename to packages/core/bootstrap/src/lib/middleware/rate-limit/index.ts index fd5c155e32..45ace637a1 100644 --- a/packages/core/bootstrap/src/lib/rate-limit/index.ts +++ b/packages/core/bootstrap/src/lib/middleware/rate-limit/index.ts @@ -1,6 +1,6 @@ import { AdapterRequest, Middleware } from '@chainlink/types' import { Store } from 'redux' -import { getHashOpts, hash } from '../util' +import { getHashOpts, hash } from '../../util' import { successfulResponseObserved } from './actions' import { Config } from './config' import * as metrics from './metrics' diff --git a/packages/core/bootstrap/src/lib/rate-limit/metrics.ts b/packages/core/bootstrap/src/lib/middleware/rate-limit/metrics.ts similarity index 100% rename from packages/core/bootstrap/src/lib/rate-limit/metrics.ts rename to packages/core/bootstrap/src/lib/middleware/rate-limit/metrics.ts diff --git a/packages/core/bootstrap/src/lib/rate-limit/reducer.ts b/packages/core/bootstrap/src/lib/middleware/rate-limit/reducer.ts similarity index 100% rename from packages/core/bootstrap/src/lib/rate-limit/reducer.ts rename to packages/core/bootstrap/src/lib/middleware/rate-limit/reducer.ts diff --git a/packages/core/bootstrap/src/lib/middleware/status-code/index.ts b/packages/core/bootstrap/src/lib/middleware/status-code/index.ts new file mode 100644 index 0000000000..88d6f74550 --- /dev/null +++ b/packages/core/bootstrap/src/lib/middleware/status-code/index.ts @@ -0,0 +1,7 @@ +import type { Middleware } from '@chainlink/types' + +export const withStatusCode: Middleware = async (execute, context) => async (input) => { + const { statusCode, data, ...rest } = await execute(input, context) + if (data && typeof data === 'object' && data.statusCode) data.statusCode = statusCode + return { ...rest, statusCode, data } +} diff --git a/packages/core/bootstrap/src/lib/ws/actions.ts b/packages/core/bootstrap/src/lib/middleware/ws/actions.ts similarity index 99% rename from packages/core/bootstrap/src/lib/ws/actions.ts rename to packages/core/bootstrap/src/lib/middleware/ws/actions.ts index 089ef4f284..cd881fcfc3 100644 --- a/packages/core/bootstrap/src/lib/ws/actions.ts +++ b/packages/core/bootstrap/src/lib/middleware/ws/actions.ts @@ -1,7 +1,7 @@ import { AdapterContext, AdapterRequest, WSHandler } from '@chainlink/types' import { createAction } from '@reduxjs/toolkit' import { WebSocketSubject } from 'rxjs/webSocket' -import { asAction } from '../store' +import { asAction } from '../../store' import { WSConfig, WSConnectionInfo } from './types' /** CONNECTIONS */ diff --git a/packages/core/bootstrap/src/lib/ws/config.ts b/packages/core/bootstrap/src/lib/middleware/ws/config.ts similarity index 98% rename from packages/core/bootstrap/src/lib/ws/config.ts rename to packages/core/bootstrap/src/lib/middleware/ws/config.ts index 3ae3b9caa0..e1f36a8963 100644 --- a/packages/core/bootstrap/src/lib/ws/config.ts +++ b/packages/core/bootstrap/src/lib/middleware/ws/config.ts @@ -1,4 +1,4 @@ -import { getEnv, parseBool } from '../util' +import { getEnv, parseBool } from '../../util' import { WSConfig } from './types' const ENV_WS_ENABLED = 'WS_ENABLED' diff --git a/packages/core/bootstrap/src/lib/ws/epics.ts b/packages/core/bootstrap/src/lib/middleware/ws/epics.ts similarity index 99% rename from packages/core/bootstrap/src/lib/ws/epics.ts rename to packages/core/bootstrap/src/lib/middleware/ws/epics.ts index 00f447d4cf..af1c813187 100644 --- a/packages/core/bootstrap/src/lib/ws/epics.ts +++ b/packages/core/bootstrap/src/lib/middleware/ws/epics.ts @@ -18,8 +18,8 @@ import { import { webSocket } from 'rxjs/webSocket' import WebSocket from 'ws' import { withCache } from '../cache' -import { censor, logger } from '../external-adapter' -import { getFeedId } from '../metrics/util' +import { censor, logger } from '../../modules' +import { getFeedId } from '../../metrics/util' import { connectFailed, connectFulfilled, @@ -55,7 +55,7 @@ import { import { getSubsId, RootState } from './reducer' import { separateBatches } from './utils' import { getWSConfig } from './config' -import { util } from '../..' +import { util } from '../../..' // Rxjs deserializer defaults to JSON.parse. // We need to handle errors from non-parsable messages diff --git a/packages/core/bootstrap/src/lib/ws/index.ts b/packages/core/bootstrap/src/lib/middleware/ws/index.ts similarity index 100% rename from packages/core/bootstrap/src/lib/ws/index.ts rename to packages/core/bootstrap/src/lib/middleware/ws/index.ts diff --git a/packages/core/bootstrap/src/lib/ws/metrics.ts b/packages/core/bootstrap/src/lib/middleware/ws/metrics.ts similarity index 100% rename from packages/core/bootstrap/src/lib/ws/metrics.ts rename to packages/core/bootstrap/src/lib/middleware/ws/metrics.ts diff --git a/packages/core/bootstrap/src/lib/ws/reducer.ts b/packages/core/bootstrap/src/lib/middleware/ws/reducer.ts similarity index 98% rename from packages/core/bootstrap/src/lib/ws/reducer.ts rename to packages/core/bootstrap/src/lib/middleware/ws/reducer.ts index 99d73b9913..87f4c48b5d 100644 --- a/packages/core/bootstrap/src/lib/ws/reducer.ts +++ b/packages/core/bootstrap/src/lib/middleware/ws/reducer.ts @@ -1,7 +1,7 @@ import { AdapterContext, AdapterRequest } from '@chainlink/types' import { combineReducers, createReducer, isAnyOf } from '@reduxjs/toolkit' -import { logger } from '../external-adapter' -import { getHashOpts, hash } from '../util' +import { logger } from '../../modules' +import { getHashOpts, hash } from '../../util' import * as actions from './actions' /** diff --git a/packages/core/bootstrap/src/lib/ws/types.ts b/packages/core/bootstrap/src/lib/middleware/ws/types.ts similarity index 100% rename from packages/core/bootstrap/src/lib/ws/types.ts rename to packages/core/bootstrap/src/lib/middleware/ws/types.ts diff --git a/packages/core/bootstrap/src/lib/ws/utils.ts b/packages/core/bootstrap/src/lib/middleware/ws/utils.ts similarity index 100% rename from packages/core/bootstrap/src/lib/ws/utils.ts rename to packages/core/bootstrap/src/lib/middleware/ws/utils.ts diff --git a/packages/core/bootstrap/src/lib/external-adapter/README.md b/packages/core/bootstrap/src/lib/modules/README.md similarity index 93% rename from packages/core/bootstrap/src/lib/external-adapter/README.md rename to packages/core/bootstrap/src/lib/modules/README.md index a080083ce6..dd52bc685b 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/README.md +++ b/packages/core/bootstrap/src/lib/modules/README.md @@ -16,7 +16,7 @@ const { Requester, Validator } = require('@chainlink/ea-bootstrap') ## Validator -Input parameter configuration can be given to the Validator in order to ensure that the requester supplied parameters which are expected by the endpoint. +Input parameter configuration can be given to the Validator in order to ensure that the Requester supplied parameters which are expected by the endpoint. Here is the type for configuration options (all attributes optional): @@ -67,7 +67,7 @@ const inputParameters: InputParameters = { ### Validator -The Validator relies on the data supplied in the inputParameters object to ensure that a requester supplied the expected parameters. +The Validator relies on the data supplied in the inputParameters object to ensure that a Requester supplied the expected parameters. #### Arguments @@ -75,7 +75,7 @@ The Validator relies on the data supplied in the inputParameters object to ensur - `input` (Object): The request payload from the Chainlink node - `inputParameters` (Object): A configuration object as shown above -Validation of the requester's input parameters can be done by creating an instance of the Validator. +Validation of the Requester's input parameters can be done by creating an instance of the Validator. ```javascript // The input data is validated upon instantiating the Validator @@ -91,7 +91,7 @@ Validated params can be obtained from the `validator.validated` object. // it's not supplied upon invoking the external adapter, it will default // to '1' const jobRunID = validator.validated.id -// Since endpoint doesn't need to be supplied by the requester, we can +// Since endpoint doesn't need to be supplied by the Requester, we can // assign a default value const endpoint = validator.validated.data.endpoint || 'price' // We specified that one of the values in the base array should be a @@ -151,7 +151,7 @@ const tokenOverride = validator.overrideIncludesToken( ## Requester -The Requester is a wrapper around a retryable pattern for reaching out to an endpoint. It can be supplied with a customError object to describe the custom error cases which the adapter should retry fetching data even if the response was successful. +The Requester is a wrapper around a retryable pattern for reaching out to an endpoint over the HTTP protocol. It can be supplied with a customError object to describe the custom error cases which the adapter should retry fetching data even if the response was successful. ```javascript const customError = (data) => { diff --git a/packages/core/bootstrap/src/lib/external-adapter/errors.ts b/packages/core/bootstrap/src/lib/modules/error.ts similarity index 100% rename from packages/core/bootstrap/src/lib/external-adapter/errors.ts rename to packages/core/bootstrap/src/lib/modules/error.ts diff --git a/packages/core/bootstrap/src/lib/external-adapter/index.ts b/packages/core/bootstrap/src/lib/modules/index.ts similarity index 51% rename from packages/core/bootstrap/src/lib/external-adapter/index.ts rename to packages/core/bootstrap/src/lib/modules/index.ts index 9f38121132..636b28497f 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/index.ts +++ b/packages/core/bootstrap/src/lib/modules/index.ts @@ -1,6 +1,5 @@ +export * from './error' export * from './requester' -export * from './validator' -export * from './errors' export * from './logger' -export * from './config' -export * from './builder' +export * from './selector' +export * from './validator' diff --git a/packages/core/bootstrap/src/lib/external-adapter/logger.ts b/packages/core/bootstrap/src/lib/modules/logger.ts similarity index 96% rename from packages/core/bootstrap/src/lib/external-adapter/logger.ts rename to packages/core/bootstrap/src/lib/modules/logger.ts index 8d07ce4646..3e1a892329 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/logger.ts +++ b/packages/core/bootstrap/src/lib/modules/logger.ts @@ -1,6 +1,6 @@ import { uuid } from '../util' import pino from 'pino' -import { wsRedactPaths } from '../ws/config' +import { wsRedactPaths } from '../middleware/ws/config' import { cloneDeep } from 'lodash' export const paths = [...wsRedactPaths] diff --git a/packages/core/bootstrap/src/lib/external-adapter/requester.ts b/packages/core/bootstrap/src/lib/modules/requester.ts similarity index 98% rename from packages/core/bootstrap/src/lib/external-adapter/requester.ts rename to packages/core/bootstrap/src/lib/modules/requester.ts index a916cc872f..d9103d8357 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/requester.ts +++ b/packages/core/bootstrap/src/lib/modules/requester.ts @@ -6,11 +6,11 @@ import { AdapterRequestData, ResultPath, } from '@chainlink/types' -import { reducer } from '../cache-warmer' +import { reducer } from '../middleware/cache-warmer' import axios, { AxiosResponse } from 'axios' import { deepType } from '../util' -import { getDefaultConfig, logConfig } from './config' -import { AdapterError } from './errors' +import { getDefaultConfig, logConfig } from '../config' +import { AdapterError } from './error' import { logger } from './logger' import objectPath from 'object-path' import { join } from 'path' diff --git a/packages/core/bootstrap/src/lib/external-adapter/builder.ts b/packages/core/bootstrap/src/lib/modules/selector.ts similarity index 99% rename from packages/core/bootstrap/src/lib/external-adapter/builder.ts rename to packages/core/bootstrap/src/lib/modules/selector.ts index 2ae7fd8498..5e07d3818b 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/builder.ts +++ b/packages/core/bootstrap/src/lib/modules/selector.ts @@ -8,7 +8,7 @@ import { AdapterContext, MakeResultPath, } from '@chainlink/types' -import { logger } from '../external-adapter' +import { logger } from '../modules' export const baseInputParameters: InputParameters = { endpoint: { diff --git a/packages/core/bootstrap/src/lib/external-adapter/validator.ts b/packages/core/bootstrap/src/lib/modules/validator.ts similarity index 97% rename from packages/core/bootstrap/src/lib/external-adapter/validator.ts rename to packages/core/bootstrap/src/lib/modules/validator.ts index c29a30b254..e2a747ed3c 100644 --- a/packages/core/bootstrap/src/lib/external-adapter/validator.ts +++ b/packages/core/bootstrap/src/lib/modules/validator.ts @@ -11,12 +11,12 @@ import { } from '@chainlink/types' import { merge } from 'lodash' import { isArray, isObject } from '../util' -import { AdapterError } from './errors' -import presetSymbols from './overrides/presetSymbols.json' -import presetTokens from './overrides/presetTokens.json' -import presetIncludes from './overrides/presetIncludes.json' +import { AdapterError } from './error' +import presetSymbols from '../config/overrides/presetSymbols.json' +import presetTokens from '../config/overrides/presetTokens.json' +import presetIncludes from '../config/overrides/presetIncludes.json' import { Requester } from './requester' -import { baseInputParameters } from './builder' +import { baseInputParameters } from './selector' export type OverrideType = 'overrides' | 'tokenOverrides' | 'includes' diff --git a/packages/core/bootstrap/src/lib/server.ts b/packages/core/bootstrap/src/lib/server.ts index ce96756bbd..be60c4c6ac 100644 --- a/packages/core/bootstrap/src/lib/server.ts +++ b/packages/core/bootstrap/src/lib/server.ts @@ -6,17 +6,17 @@ import rateLimit from 'express-rate-limit' import { join } from 'path' import * as client from 'prom-client' import { executeSync, storeSlice, withMiddleware } from '../index' -import { defaultOptions } from './cache' +import { defaultOptions } from './middleware/cache' import { loadTestPayload } from './config/test-payload-loader' import { HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE, HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE_MESSAGE, } from './errors' -import { logger } from './external-adapter' +import { logger } from './modules' import { METRICS_ENABLED, httpRateLimit, setupMetrics } from './metrics' -import { get as getRateLimitConfig } from './rate-limit/config' +import { get as getRateLimitConfig } from './middleware/rate-limit/config' import { toObjectWithNumbers } from './util' -import { warmupShutdown } from './cache-warmer/actions' +import { warmupShutdown } from './middleware/cache-warmer/actions' import { AddressInfo } from 'net' const app = express() diff --git a/packages/core/bootstrap/src/lib/util.ts b/packages/core/bootstrap/src/lib/util.ts index 0409349049..7a22bbdf2d 100644 --- a/packages/core/bootstrap/src/lib/util.ts +++ b/packages/core/bootstrap/src/lib/util.ts @@ -3,7 +3,7 @@ import { Decimal } from 'decimal.js' import { flatMap, values, pick, omit } from 'lodash' import objectHash from 'object-hash' import { v4 as uuidv4 } from 'uuid' -import { CacheEntry } from './cache/types' +import { CacheEntry } from './middleware/cache/types' export const isObject = (o: unknown): boolean => o !== null && typeof o === 'object' && Array.isArray(o) === false diff --git a/packages/core/bootstrap/test/unit/cache-warmer.test.ts b/packages/core/bootstrap/test/unit/cache-warmer.test.ts index 86f6c626fc..c6ff2e8d68 100644 --- a/packages/core/bootstrap/test/unit/cache-warmer.test.ts +++ b/packages/core/bootstrap/test/unit/cache-warmer.test.ts @@ -5,17 +5,17 @@ import { Subject } from 'rxjs' import { RunHelpers } from 'rxjs/internal/testing/TestScheduler' import { TestScheduler } from 'rxjs/testing' import { stub } from 'sinon' -import * as actions from '../../src/lib/cache-warmer/actions' -import { get } from '../../src/lib/cache-warmer/config' +import * as actions from '../../src/lib/middleware/cache-warmer/actions' +import { get } from '../../src/lib/middleware/cache-warmer/config' import { EpicDependencies, executeHandler, warmupRequestHandler, warmupSubscriber, warmupUnsubscriber, -} from '../../src/lib/cache-warmer/epics' -import { subscriptionsReducer } from '../../src/lib/cache-warmer/reducer' -import { RootState, SubscriptionState } from '../../src/lib/cache-warmer/reducer' +} from '../../src/lib/middleware/cache-warmer/epics' +import { subscriptionsReducer } from '../../src/lib/middleware/cache-warmer/reducer' +import { RootState, SubscriptionState } from '../../src/lib/middleware/cache-warmer/reducer' let scheduler: TestScheduler diff --git a/packages/core/bootstrap/test/unit/cache.test.ts b/packages/core/bootstrap/test/unit/cache.test.ts index 761ec002e5..299feac863 100644 --- a/packages/core/bootstrap/test/unit/cache.test.ts +++ b/packages/core/bootstrap/test/unit/cache.test.ts @@ -1,7 +1,7 @@ import { AdapterContext, Execute } from '@chainlink/types' import { useFakeTimers } from 'sinon' -import { defaultOptions, withCache } from '../../src/lib/cache' -import { LocalLRUCache } from '../../src/lib/cache/local' +import { defaultOptions, withCache } from '../../src/lib/middleware/cache' +import { LocalLRUCache } from '../../src/lib/middleware/cache/local' const callAndExpect = async (fn: any, n: number, result: any) => { while (n--) { diff --git a/packages/core/bootstrap/test/unit/config.test.ts b/packages/core/bootstrap/test/unit/config.test.ts index f19a7bd538..b6d5b9feaa 100644 --- a/packages/core/bootstrap/test/unit/config.test.ts +++ b/packages/core/bootstrap/test/unit/config.test.ts @@ -1,5 +1,5 @@ -import { constants } from '../../src/lib/external-adapter/config' -import { Requester } from '../../src/lib/external-adapter/requester' +import { constants } from '../../src/lib/config' +import { Requester } from '../../src/lib/modules/requester' const { ENV_API_ENDPOINT, ENV_API_KEY, ENV_API_TIMEOUT, DEFAULT_API_TIMEOUT } = constants describe('incorrect app config', () => { diff --git a/packages/core/bootstrap/test/unit/overrides.test.ts b/packages/core/bootstrap/test/unit/overrides.test.ts index 2e23e52cd8..a81f64b20d 100644 --- a/packages/core/bootstrap/test/unit/overrides.test.ts +++ b/packages/core/bootstrap/test/unit/overrides.test.ts @@ -1,4 +1,4 @@ -import presetSymbols from '../../src/lib/external-adapter/overrides/presetSymbols.json' +import presetSymbols from '../../src/lib/config/overrides/presetSymbols.json' describe('presetSymbols', () => { it('Should not contain any overrides that lead to another override', () => { diff --git a/packages/core/bootstrap/test/unit/provider-limits/limits.test.ts b/packages/core/bootstrap/test/unit/provider-limits/limits.test.ts index 9f49437d8a..b63ad76856 100644 --- a/packages/core/bootstrap/test/unit/provider-limits/limits.test.ts +++ b/packages/core/bootstrap/test/unit/provider-limits/limits.test.ts @@ -1,6 +1,6 @@ -import * as limits from '../../../src/lib/provider-limits' +import * as limits from '../../../src/lib/config/provider-limits' -const limitsJSONPath = '../../../src/lib/provider-limits/limits.json' +const limitsJSONPath = '../../../src/lib/config/provider-limits/limits.json' jest.mock( '../../../src/lib/provider-limits/limits.json', @@ -31,15 +31,15 @@ jest.mock( describe('Provider Limits', () => { describe('Limits API', () => { it('gets the correct rate limits', () => { - const limit = limits.getRateLimit('amberdata', 'starter') - expect(limit.minute).toBe(10 / 60) - expect(limit.second).toBe((10 / 60 / 60) * 2) + const limit = limits.getRateLimit('amberdata', 'free') + expect(limit.minute).toBe(83.33 / 60) + expect(limit.second).toBe(1) }) it('rate limit defaults to lowest tier if no tier match', () => { const limit = limits.getRateLimit('amberdata', 'non-existant') - expect(limit.minute).toBe(10 / 60) - expect(limit.second).toBe((10 / 60 / 60) * 2) + expect(limit.minute).toBe(83.33 / 60) + expect(limit.second).toBe(1) }) it('rate limit throws if no provider match', () => { @@ -49,15 +49,15 @@ describe('Provider Limits', () => { }) it('gets the correct ws limits', () => { - const limit = limits.getWSLimits('amberdata', 'starter') - expect(limit.connections).toBe(10) - expect(limit.subscriptions).toBe(20) + const limit = limits.getWSLimits('amberdata', 'free') + expect(limit.connections).toBe(1) + expect(limit.subscriptions).toBe(10) }) - it('WS limit throws if no tier match', () => { + it('WS defaults to lowest tier if no tier match', () => { const limit = limits.getWSLimits('amberdata', 'non-existant') - expect(limit.connections).toBe(10) - expect(limit.subscriptions).toBe(20) + expect(limit.connections).toBe(1) + expect(limit.subscriptions).toBe(10) }) it('WS limit throws if no provider match', () => { diff --git a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/1-rate-limit-cache.test.ts b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/1-rate-limit-cache.test.ts index d6c38574ea..66d9f72714 100644 --- a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/1-rate-limit-cache.test.ts +++ b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/1-rate-limit-cache.test.ts @@ -1,10 +1,10 @@ import { createStore } from 'redux' import { stub } from 'sinon' -import { withDebug } from '../../../src' -import { defaultOptions, withCache } from '../../../src/lib/cache' -import { logger } from '../../../src/lib/external-adapter' -import * as rateLimit from '../../../src/lib/rate-limit' -import { get } from '../../../src/lib/rate-limit/config' +import { withDebug } from '../../../src/lib/middleware/debugger' +import { defaultOptions, withCache } from '../../../src/lib/middleware/cache' +import { logger } from '../../../src/lib/modules' +import * as rateLimit from '../../../src/lib/middleware/rate-limit' +import { get } from '../../../src/lib/middleware/rate-limit/config' import { dataProviderMock, getRLTokenSpentPerMinute, @@ -57,14 +57,12 @@ describe('Rate Limit/Cache - Integration', () => { rateLimit.withRateLimit(store), withDebug, ]) - const secsInMin = 60 for (let i = 0; i < secsInMin; i++) { const input = { id: '6', data: { test1: 1 } } await executeWithMiddleware(input, context) clock.tick(1000) } - const state = store.getState() const rlPerMinute = getRLTokenSpentPerMinute(state.heartbeats) const minute = cost - 1 @@ -149,46 +147,46 @@ describe('Rate Limit/Cache - Integration', () => { restoreClock() }) - // it('1 h simulation', async () => { - // const [clock, restoreClock] = setupClock() - // const dataProvider = dataProviderMock() - // const store = newStore() - // const executeWithWarmer = await makeExecuteWithWarmer(dataProvider.execute, store) - - // // 120 Feeds: 3 Composite, rest are single feeds - // const totalFeeds = 120 - // const composite = 3 - // const feeds = new Array(totalFeeds).fill('').map((_, feedId) => { - // if (feedId % (totalFeeds / composite) === 0) { - // return new Array(10).fill('').map((_, internalReq) => { - // return { id: '6', data: { singleFeed: feedId, quote: internalReq } } - // }) - // } - // return [{ id: '6', data: { singleFeed: feedId, quote: 1 } }] - // }) - - // const _getRandomFeed = () => { - // return feeds[Math.floor(Math.random() * feeds.length)] - // } - - // const timeBetweenRequests = 1000 - // const hours = 1 - // for (let i = 0; i < (1000 / timeBetweenRequests) * hours * 60 * 60; i++) { - // const feed = _getRandomFeed() - // for (let i = 0; i < feed.length; i++) { - // const input = feed[i] - // await executeWithWarmer(input) - // } - // clock.tick(timeBetweenRequests) - // } - - // const state = store.getState() - // const rlPerMinute = getRLTokenSpentPerMinute(state.rateLimit.heartbeats) - - // Object.values(rlPerMinute).forEach((req) => { - // // TODO: check that + 20 is the right capacity - // expect(req).toBeLessThan(capacity + 20) - // }) - // restoreClock() - // }) + it('1 h simulation', async () => { + const [clock, restoreClock] = setupClock() + const dataProvider = dataProviderMock() + const store = newStore() + const executeWithWarmer = await makeExecuteWithWarmer(dataProvider.execute, store) + + // 120 Feeds: 3 Composite, rest are single feeds + const totalFeeds = 120 + const composite = 3 + const feeds = new Array(totalFeeds).fill('').map((_, feedId) => { + if (feedId % (totalFeeds / composite) === 0) { + return new Array(10).fill('').map((_, internalReq) => { + return { id: '6', data: { singleFeed: feedId, quote: internalReq } } + }) + } + return [{ id: '6', data: { singleFeed: feedId, quote: 1 } }] + }) + + const _getRandomFeed = () => { + return feeds[Math.floor(Math.random() * feeds.length)] + } + + const timeBetweenRequests = 1000 + const hours = 1 + for (let i = 0; i < (1000 / timeBetweenRequests) * hours * 60 * 60; i++) { + const feed = _getRandomFeed() + for (let i = 0; i < feed.length; i++) { + const input = feed[i] + await executeWithWarmer(input) + } + clock.tick(timeBetweenRequests) + } + + const state = store.getState() + const rlPerMinute = getRLTokenSpentPerMinute(state.rateLimit.heartbeats) + + Object.values(rlPerMinute).forEach((req) => { + // TODO: check that + 20 is the right capacity + expect(req).toBeLessThan(capacity + 20) + }) + restoreClock() + }) }) diff --git a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/2-rate-limit-cache.test.ts b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/2-rate-limit-cache.test.ts index 8afcdcd0d3..f13c1bd3bf 100644 --- a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/2-rate-limit-cache.test.ts +++ b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/2-rate-limit-cache.test.ts @@ -1,12 +1,12 @@ import { createStore } from 'redux' import { stub } from 'sinon' -import { withDebug } from '../../../src' -import { defaultOptions, withCache } from '../../../src/lib/cache' -import { logger } from '../../../src/lib/external-adapter' -import * as rateLimit from '../../../src/lib/rate-limit' -import { get } from '../../../src/lib/rate-limit/config' +import { withDebug } from '../../../src/lib/middleware/debugger' +import { defaultOptions, withCache } from '../../../src/lib/middleware/cache' +import { logger } from '../../../src/lib/modules' +import * as rateLimit from '../../../src/lib/middleware/rate-limit' +import { get } from '../../../src/lib/middleware/rate-limit/config' import { dataProviderMock, getRLTokenSpentPerMinute, setupClock } from './helpers' -import { withMiddleware } from '../../../src/index' +import { withMiddleware } from '../../../src' import { AdapterContext } from '@chainlink/types' describe('Rate Limit/Cache - Integration', () => { @@ -43,7 +43,6 @@ describe('Rate Limit/Cache - Integration', () => { it('Composite feeds requests go over capacity on initialization, then stabilize', async () => { const [clock, restoreClock] = setupClock() - const store = createStore(rateLimit.reducer.rootReducer, {}) const dataProvider = dataProviderMock() const executeWithMiddleware = await withMiddleware(dataProvider.execute, context, [ diff --git a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/3-rate-limit-cache.test.ts b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/3-rate-limit-cache.test.ts index 62f48ba619..69695c8a13 100644 --- a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/3-rate-limit-cache.test.ts +++ b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/3-rate-limit-cache.test.ts @@ -1,5 +1,5 @@ import { stub } from 'sinon' -import { logger } from '../../../src/lib/external-adapter' +import { logger } from '../../../src/lib/modules' import { dataProviderMock, getRLTokenSpentPerMinute, diff --git a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/4-rate-limit-cache.test.ts b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/4-rate-limit-cache.test.ts index 9301493491..1811864ae2 100644 --- a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/4-rate-limit-cache.test.ts +++ b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/4-rate-limit-cache.test.ts @@ -1,10 +1,10 @@ import { createStore } from 'redux' import { stub } from 'sinon' -import { withDebug } from '../../../src' -import { defaultOptions, withCache } from '../../../src/lib/cache' -import { logger } from '../../../src/lib/external-adapter' -import * as rateLimit from '../../../src/lib/rate-limit' -import { get } from '../../../src/lib/rate-limit/config' +import { withDebug } from '../../../src/lib/middleware/debugger' +import { defaultOptions, withCache } from '../../../src/lib/middleware/cache' +import { logger } from '../../../src/lib/modules' +import * as rateLimit from '../../../src/lib/middleware/rate-limit' +import { get } from '../../../src/lib/middleware/rate-limit/config' import { dataProviderMock, getRLTokenSpentPerMinute, setupClock } from './helpers' import { withMiddleware } from '../../../src/index' import { AdapterContext } from '@chainlink/types' diff --git a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/helpers.ts b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/helpers.ts index c342af7296..b5dbc86e6c 100644 --- a/packages/core/bootstrap/test/unit/rate-limit-cache-tests/helpers.ts +++ b/packages/core/bootstrap/test/unit/rate-limit-cache-tests/helpers.ts @@ -1,13 +1,12 @@ import { AdapterContext, AdapterRequest, Execute } from '@chainlink/types' -import { combineReducers, Store } from 'redux' +import { createStore, combineReducers, Store } from 'redux' import { useFakeTimers } from 'sinon' -import { withDebug } from '../../../src' -import { defaultOptions, withCache } from '../../../src/lib/cache' -import * as cacheWarmer from '../../../src/lib/cache-warmer' -import * as rateLimit from '../../../src/lib/rate-limit' -import { get } from '../../../src/lib/rate-limit/config' -import { configureStore } from '../../../src/lib/store' import { withMiddleware } from '../../../src/index' +import { withDebug } from '../../../src/lib/middleware/debugger' +import { defaultOptions, withCache } from '../../../src/lib/middleware/cache' +import * as cacheWarmer from '../../../src/lib/middleware/cache-warmer' +import * as rateLimit from '../../../src/lib/middleware/rate-limit' +import { get } from '../../../src/lib/middleware/rate-limit/config' export const newStore = () => { const initState = { cacheWarmer: {}, rateLimit: {} } @@ -15,8 +14,8 @@ export const newStore = () => { cacheWarmer: cacheWarmer.reducer.rootReducer, rateLimit: rateLimit.reducer.rootReducer, }) - cacheWarmer.epics.epicMiddleware.run(cacheWarmer.epics.rootEpic) - return configureStore(rootReducer, initState, [cacheWarmer.epics.epicMiddleware]) + const store = createStore(rootReducer, initState) + return store } export const makeExecuteWithWarmer = async (execute: Execute, store: Store) => { diff --git a/packages/core/bootstrap/test/unit/rate-limit.test.ts b/packages/core/bootstrap/test/unit/rate-limit.test.ts index b89def26ba..327a3eae4a 100644 --- a/packages/core/bootstrap/test/unit/rate-limit.test.ts +++ b/packages/core/bootstrap/test/unit/rate-limit.test.ts @@ -1,14 +1,14 @@ import { AdapterRequest, AdapterContext, Execute } from '@chainlink/types' import { createStore, Store } from 'redux' import { useFakeTimers } from 'sinon' -import * as rateLimit from '../../src/lib/rate-limit' -import { Config, get } from '../../src/lib/rate-limit/config' +import * as rateLimit from '../../src/lib/middleware/rate-limit' +import { Config, get } from '../../src/lib/middleware/rate-limit/config' import { IntervalNames, Intervals, selectParticiantsHeartbeatsFor, selectTotalNumberOfHeartbeatsFor, -} from '../../src/lib/rate-limit/reducer' +} from '../../src/lib/middleware/rate-limit/reducer' const counterFrom = (i = 0): Execute => diff --git a/packages/core/bootstrap/test/unit/requester.test.ts b/packages/core/bootstrap/test/unit/requester.test.ts index dc553a4c5f..544f0785b8 100644 --- a/packages/core/bootstrap/test/unit/requester.test.ts +++ b/packages/core/bootstrap/test/unit/requester.test.ts @@ -1,7 +1,7 @@ -import { Requester } from '../../src/lib/external-adapter/requester' +import { Requester } from '../../src/lib/modules/requester' import { Server, SUCCESS_ARRAY_RESPONSE } from '../helpers/server' -describe('Requester', () => { +describe('HTTP', () => { const errorMessage = 'Request failed with status code 500' const customErrorMessage = 'Could not retrieve valid data: {"result":"error","value":1}' const successUrl = 'http://localhost:18080' diff --git a/packages/core/bootstrap/test/unit/validator.test.ts b/packages/core/bootstrap/test/unit/validator.test.ts index 09129f0142..21ac3ab6d5 100644 --- a/packages/core/bootstrap/test/unit/validator.test.ts +++ b/packages/core/bootstrap/test/unit/validator.test.ts @@ -1,5 +1,5 @@ import { InputParameters } from '@chainlink/types' -import { Validator } from '../../src/lib/external-adapter/validator' +import { Validator } from '../../src/lib/modules/validator' describe('Validator', () => { describe('with no input parameter configuration', () => { diff --git a/packages/core/bootstrap/test/unit/ws.test.ts b/packages/core/bootstrap/test/unit/ws.test.ts index d88c498869..efcd5d219c 100644 --- a/packages/core/bootstrap/test/unit/ws.test.ts +++ b/packages/core/bootstrap/test/unit/ws.test.ts @@ -1,4 +1,4 @@ -import { separateBatches } from '../../src/lib/ws/utils' +import { separateBatches } from '../../src/lib/middleware/ws/utils' describe('WebSockets', () => { describe('Separate batches', () => {