133/prometheus

README.md

@@ -148,6 +148,25 @@ There are 3 files that live in the relayer's home.
 
 [Learn more about configuration.](spec/config.md)
 
+## Monitoring
+The relayer collects various metrics that [Prometheus](https://prometheus.io/docs/introduction/overview/) instance can consume.
+
+To enable metrics collection, pass `--enable-metrics` flag when starting the relayer:
+```sh
+ibc-relayer start --enable-metrics
+```
+> **NOTE:** Metrics can be also enabled via an environment variable `RELAYER_ENABLE_METRICS=true`, or with a `enableMetrics: true` entry in the `app.yaml` file, as explained in the [config specifiation](./spec/config.md#configuration).
+
+The `GET /metrics` endpoint is exposed on a default port `26660`, which you can modify with `--metrics-port` flag/env variable/entry in app.yaml.
+
+### Local setup
+1. Start the relayer with metrics enabled
+2. Spin up the Prometheus instance:
+   ```sh
+   docker run -it -v $(pwd):/prometheus  -p9090:9090  prom/prometheus --config.file=prometheus.yaml
+   ```
+   > **NOTE:** Ensure that `--config.file=<path>` flag points at the existing configuration file, you can find an example here: [prometheus.yaml](prometheus.yaml).
+3. Open Prometheus dashboard in the browser at [http://localhost:9090](http://localhost:9090)
 ## Development
 [Refer to the development page.](DEVELOPMENT.md)
 

package.json

@@ -43,6 +43,7 @@
     "fast-safe-stringify": "2.0.4",
     "js-yaml": "4.0.0",
     "lodash": "4.17.21",
+    "prom-client": "13.1.0",
     "protobufjs": "6.10.2",
     "table": "6.0.7",
     "triple-beam": "1.3.0",

prometheus.yaml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: confio_relayer
+
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['host.docker.internal:26660']

src/binary/ibc-relayer/commands/start.ts

@@ -16,6 +16,7 @@ import { resolveHomeOption } from '../../utils/options/shared/resolve-home-optio
 import { resolveKeyFileOption } from '../../utils/options/shared/resolve-key-file-option';
 import { resolveMnemonicOption } from '../../utils/options/shared/resolve-mnemonic-option';
 import { signingClient } from '../../utils/signing-client';
+import { Metrics, setupPrometheus } from '../setup-prometheus';
 
 type ResolveHeightsParams = {
   scanFromSrc: number | null;
@@ -94,6 +95,8 @@ type Flags = {
   destConnection?: string;
   scanFromSrc?: string;
   scanFromDest?: string;
+  enableMetrics: boolean;
+  metricsPort?: string;
 } & LoggerFlags &
   LoopFlags;
 
@@ -105,15 +108,17 @@ type Options = {
   srcConnection: string;
   destConnection: string;
   heights: RelayedHeights | null;
+  enableMetrics: boolean;
+  metricsPort: number;
 } & LoopOptions;
 
-// some defaults for looping
 export const defaults = {
   // check once per minute
   poll: 60,
   // once per day: 86400s
   maxAgeSrc: 86400,
   maxAgeDest: 86400,
+  metricsPort: 26660,
 };
 
 export async function start(flags: Flags, logger: Logger) {
@@ -171,6 +176,20 @@ export async function start(flags: Flags, logger: Logger) {
     flags.scanFromDest,
     process.env.RELAYER_SCAN_FROM_DEST
   );
+  const enableMetrics =
+    flags.enableMetrics ||
+    Boolean(process.env.RELAYER_ENABLE_METRICS) ||
+    app?.enableMetrics ||
+    false;
+  const metricsPort = resolveOption('metricsPort', {
+    integer: true,
+    required: true,
+  })(
+    flags.metricsPort,
+    process.env.RELAYER_METRICS_PORT,
+    app?.metricsPort,
+    defaults.metricsPort
+  );
 
   const heights = resolveHeights({ scanFromSrc, scanFromDest, home }, logger);
 
@@ -189,12 +208,20 @@ export async function start(flags: Flags, logger: Logger) {
     maxAgeDest,
     once,
     heights,
+    enableMetrics,
+    metricsPort,
   };
 
   await run(options, logger);
 }
 
 async function run(options: Options, logger: Logger) {
+  const metrics = setupPrometheus({
+    enabled: options.enableMetrics,
+    port: options.metricsPort,
+    logger,
+  });
+
   const registryFilePath = path.join(options.home, registryFile);
   const { chains } = loadAndValidateRegistry(registryFilePath);
   const srcChain = chains[options.src];
@@ -224,10 +251,15 @@ async function run(options: Options, logger: Logger) {
     logger
   );
 
-  await relayerLoop(link, options, logger);
+  await relayerLoop(link, options, logger, metrics);
 }
 
-async function relayerLoop(link: Link, options: Options, logger: Logger) {
+async function relayerLoop(
+  link: Link,
+  options: Options,
+  logger: Logger,
+  metrics: Metrics
+) {
   let nextRelay = options.heights ?? {};
   const lastQueriedHeightsFilePath = path.join(
     options.home,
@@ -262,5 +294,7 @@ async function relayerLoop(link: Link, options: Options, logger: Logger) {
     logger.info(`Sleeping ${options.poll} seconds...`);
     await sleep(options.poll * 1000);
     logger.info('... waking up and checking for packets!');
+
+    metrics?.pollCounter?.inc();
   }
 }

src/binary/ibc-relayer/index.ts

@@ -38,6 +38,13 @@ const startCommand = program
   .addOption(mnemonicOption)
   .addOption(srcConnection)
   .addOption(destConnection)
+  .option(
+    '--enable-metrics',
+    'Enable Prometheus metrics collection and GET /metrics endpoint'
+  )
+  .option(
+    `--metrics-port <port>', 'Specify port for GET /metrics http server (default: ${startDefaults.metricsPort})`
+  )
   .option(
     '--poll <frequency>',
     `How many seconds we sleep between checking for packets (default: ${startDefaults.poll})`

src/binary/ibc-relayer/setup-prometheus.ts

@@ -0,0 +1,58 @@
+import http from 'http';
+
+import client from 'prom-client';
+
+import { Logger } from '../create-logger';
+
+let initialized = false;
+
+function getMetrics() {
+  return {
+    pollCounter: new client.Counter({
+      name: 'poll_counter',
+      help: 'Poll counter',
+    }),
+  };
+}
+
+export type Metrics = ReturnType<typeof setupPrometheus>;
+export function setupPrometheus({
+  enabled,
+  port,
+  logger,
+}: {
+  enabled: boolean;
+  port: number;
+  logger: Logger;
+}) {
+  if (initialized) {
+    throw new Error(
+      `"setupPrometheus" func shouldn't be initialized more than once.`
+    );
+  }
+  initialized = true;
+
+  if (!enabled) {
+    return null;
+  }
+
+  client.collectDefaultMetrics({ prefix: 'confio_relayer_' });
+  const server = http.createServer(async (request, response) => {
+    if (request.method === 'GET' && request.url === '/metrics') {
+      const metrics = await client.register.metrics();
+      response.writeHead(200, {
+        'Content-Type': client.register.contentType,
+      });
+      response.end(metrics);
+
+      return;
+    }
+
+    response.writeHead(404);
+    response.end('404');
+  });
+  server.listen(port);
+  logger.info(`Prometheus GET /metrics exposed on port ${port}.`);
+
+  return getMetrics();
+}

src/binary/types.ts

@@ -22,6 +22,8 @@ export type AppConfig = {
   destConnection?: string;
   mnemonic?: string;
   keyFile?: string;
+  enableMetrics?: boolean;
+  metricsPort?: number;
 };
 
 export type LoggerFlags = {

src/binary/utils/load-and-validate-app.ts

@@ -41,6 +41,8 @@ export function loadAndValidateApp(home: string) {
       destConnection: { type: 'string', nullable: true },
       mnemonic: { type: 'string', nullable: true },
       keyFile: { type: 'string', nullable: true },
+      enableMetrics: { type: 'boolean', nullable: true },
+      metricsPort: { type: 'number', nullable: true },
     },
   };
   const validate = ajv.compile(schema);

yarn.lock

@@ -958,6 +958,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+[email protected]:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524"
+  integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=
+
 bip39@^3.0.2:
   version "3.0.3"
   resolved "https://registry.npmjs.org/bip39/-/bip39-3.0.3.tgz"
@@ -3513,6 +3518,13 @@ progress@^2.0.0:
   resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
+[email protected]:
+  version "13.1.0"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-13.1.0.tgz#1185caffd8691e28d32e373972e662964e3dba45"
+  integrity sha512-jT9VccZCWrJWXdyEtQddCDszYsiuWj5T0ekrPszi/WEegj3IZy6Mm09iOOVM86A4IKMWq8hZkT2dD9MaSe+sng==
+  dependencies:
+    tdigest "^0.1.1"
+
 [email protected], protobufjs@^6.8.8, protobufjs@~6.10.2:
   version "6.10.2"
   resolved "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz"
@@ -4141,6 +4153,13 @@ [email protected], table@^6.0.4:
     slice-ansi "^4.0.0"
     string-width "^4.2.0"
 
+tdigest@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021"
+  integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=
+  dependencies:
+    bintrees "1.0.1"
+
 [email protected]:
   version "6.0.1"
   resolved "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.1.tgz"