133/prometheus

README.md

@@ -14,7 +14,6 @@ the relayer works, but the Quick Start probably gives a better intro.
 - RPC addresses of 2 full nodes on compatible, IBC-enabled chains
 - See [Chain Requirements below](#Chain-Requirements) for details of what chains are supported
 
-
 ## Installation
 
 ### NPM
@@ -66,7 +65,7 @@ Reads the configuration and starts relaying packets.
    - pulls default `registry.yaml` to relayer's home
    - funds addresses on both sides so relayer can pay the fee while relaying packets
 
-   > **NOTE:** Test blockchains `relayer_test_1` and `relayer_test_2` are  running in the public. You do not need to start any blockchain locally to complete the quick start guide.
+   > **NOTE:** Test blockchains `relayer_test_1` and `relayer_test_2` are running in the public. You do not need to start any blockchain locally to complete the quick start guide.
 
    > **NOTE:** Run `ibc-setup balances` to see the amount of tokens on each address.
 
@@ -88,15 +87,16 @@ Reads the configuration and starts relaying packets.
 ### Send tokens between chains
 
 1. Make sure `wasmd` binary is installed on your system
+
    - you must be running Linux or OSX on amd64 (not arm64/Mac M1)
    - [install Go 1.15+](https://golang.org/doc/install) and ensure that `$PATH` includes Go binaries (you may need to restart your terminal session)
-   -  clone and install `wasmd`:
-      ```sh
-      git clone https://github.com/CosmWasm/wasmd.git
-      cd wasmd
-      git checkout v0.15.1
-      make install
-      ```
+   - clone and install `wasmd`:
+     ```sh
+     git clone https://github.com/CosmWasm/wasmd.git
+     cd wasmd
+     git checkout v0.15.1
+     make install
+     ```
 
 2. Create a new account and fund it
 
@@ -125,6 +125,7 @@ Reads the configuration and starts relaying packets.
 ## Configuration overview
 
 The relayer configuration is stored under relayer's home directory. By default, it's located at `$HOME/.ibc-setup`, however, can be customized with `home` option, e.g.:
+
 ```sh
 # initialize the configuration at /home/user/relayer_custom_home
 ibc-setup init --home /home/user/relayer_custom_home
@@ -136,19 +137,45 @@ ibc-relayer start --home /home/user/relayer_custom_home
 There are 3 files that live in the relayer's home.
 
 - **registry.yaml** (required)
-  
-   Contains a list of available chains with corresponding information. The chains from the registry can be referenced by `ibc-setup` binary or within the `app.yaml` file. [View an example of registry.yaml file.](demo/registry.yaml)
+
+  Contains a list of available chains with corresponding information. The chains from the registry can be referenced by `ibc-setup` binary or within the `app.yaml` file. [View an example of registry.yaml file.](demo/registry.yaml)
 
 - **app.yaml** (optional)
-
-   Holds the relayer-specific options such as source or destination chains. These options can be overridden with CLI flags or environment variables.
+
+  Holds the relayer-specific options such as source or destination chains. These options can be overridden with CLI flags or environment variables.
+
 - **last-queried-heights.json** (optional)
-  
+
   Stores last queried heights for better performance on relayer startup. It's constantly overwritten with new heights when relayer is running. Simply delete this file to scan the events since forever.
 
 [Learn more about configuration.](spec/config.md)
 
+## Monitoring
+
+The relayer collects various metrics that a [Prometheus](https://prometheus.io/docs/introduction/overview/) instance can consume.
+
+To enable metrics collection, pass the `--enable-metrics` flag when starting the relayer:
+
+```sh
+ibc-relayer start --enable-metrics
+```
+
+> **NOTE:** Metrics can also be enabled via an environment variable `RELAYER_ENABLE_METRICS=true`, or with an `enableMetrics: true` entry in the `app.yaml` file, as explained in the [config specification](./spec/config.md#configuration).
+
+The `GET /metrics` endpoint will be exposed by default on port `26660`, which you can override with `--metrics-port` flag, `RELAYER_METRICS_PORT` env variable, or `metricsPort` entry in `app.yaml`.
+
+### Local setup
+
+1. Start the relayer with metrics enabled
+2. Spin up the Prometheus instance:
+   ```sh
+   docker run -it -v $(pwd):/prometheus -p9090:9090 prom/prometheus --config.file=prometheus.yaml
+   ```
+   > **NOTE:** Ensure that `the --config.file=<path>` flag points at the existing configuration file. You can find an example here: [prometheus.yaml](prometheus.yaml).
+3. Open the Prometheus dashboard in a browser at [http://localhost:9090](http://localhost:9090)
+
 ## Development
+
 [Refer to the development page.](DEVELOPMENT.md)
 
 ## Chain Requirements

package.json

@@ -43,6 +43,7 @@
     "fast-safe-stringify": "2.0.4",
     "js-yaml": "4.0.0",
     "lodash": "4.17.21",
+    "prom-client": "13.1.0",
     "protobufjs": "6.10.2",
     "table": "6.0.7",
     "triple-beam": "1.3.0",

prometheus.yaml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: confio_relayer
+
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['host.docker.internal:26660']

src/binary/ibc-relayer/commands/start.ts

@@ -16,6 +16,7 @@ import { resolveHomeOption } from '../../utils/options/shared/resolve-home-optio
 import { resolveKeyFileOption } from '../../utils/options/shared/resolve-key-file-option';
 import { resolveMnemonicOption } from '../../utils/options/shared/resolve-mnemonic-option';
 import { signingClient } from '../../utils/signing-client';
+import { Metrics, setupPrometheus } from '../setup-prometheus';
 
 type ResolveHeightsParams = {
   scanFromSrc: number | null;
@@ -67,53 +68,57 @@ function resolveHeights(
 }
 
 type LoopFlags = {
-  poll?: string;
-  maxAgeSrc?: string;
-  maxAgeDest?: string;
-  once: boolean;
+  readonly poll?: string;
+  readonly maxAgeSrc?: string;
+  readonly maxAgeDest?: string;
+  readonly once: boolean;
 };
 type LoopOptions = {
   // how many seconds we sleep between relaying batches
-  poll: number;
+  readonly poll: number;
   // number of seconds old the client on chain A can be
-  maxAgeSrc: number;
+  readonly maxAgeSrc: number;
   // number of seconds old the client on chain B can be
-  maxAgeDest: number;
+  readonly maxAgeDest: number;
   // if set to 'true' quit after one pass
-  once: boolean;
+  readonly once: boolean;
 };
 
 type Flags = {
-  interactive: boolean;
-  home?: string;
-  src?: string;
-  dest?: string;
-  keyFile?: string;
-  mnemonic?: string;
-  srcConnection?: string;
-  destConnection?: string;
-  scanFromSrc?: string;
-  scanFromDest?: string;
+  readonly interactive: boolean;
+  readonly home?: string;
+  readonly src?: string;
+  readonly dest?: string;
+  readonly keyFile?: string;
+  readonly mnemonic?: string;
+  readonly srcConnection?: string;
+  readonly destConnection?: string;
+  readonly scanFromSrc?: string;
+  readonly scanFromDest?: string;
+  readonly enableMetrics: boolean;
+  readonly metricsPort?: string;
 } & LoggerFlags &
   LoopFlags;
 
 type Options = {
-  home: string;
-  src: string;
-  dest: string;
-  mnemonic: string;
-  srcConnection: string;
-  destConnection: string;
-  heights: RelayedHeights | null;
+  readonly home: string;
+  readonly src: string;
+  readonly dest: string;
+  readonly mnemonic: string;
+  readonly srcConnection: string;
+  readonly destConnection: string;
+  readonly heights: RelayedHeights | null;
+  readonly enableMetrics: boolean;
+  readonly metricsPort: number;
 } & LoopOptions;
 
-// some defaults for looping
 export const defaults = {
   // check once per minute
   poll: 60,
   // once per day: 86400s
   maxAgeSrc: 86400,
   maxAgeDest: 86400,
+  metricsPort: 26660,
 };
 
 export async function start(flags: Flags, logger: Logger) {
@@ -171,6 +176,20 @@ export async function start(flags: Flags, logger: Logger) {
     flags.scanFromDest,
     process.env.RELAYER_SCAN_FROM_DEST
   );
+  const enableMetrics =
+    flags.enableMetrics ||
+    Boolean(process.env.RELAYER_ENABLE_METRICS) ||
+    app?.enableMetrics ||
+    false;
+  const metricsPort = resolveOption('metricsPort', {
+    integer: true,
+    required: true,
+  })(
+    flags.metricsPort,
+    process.env.RELAYER_METRICS_PORT,
+    app?.metricsPort,
+    defaults.metricsPort
+  );
 
   const heights = resolveHeights({ scanFromSrc, scanFromDest, home }, logger);
 
@@ -189,12 +208,20 @@ export async function start(flags: Flags, logger: Logger) {
     maxAgeDest,
     once,
     heights,
+    enableMetrics,
+    metricsPort,
   };
 
   await run(options, logger);
 }
 
 async function run(options: Options, logger: Logger) {
+  const metrics = setupPrometheus({
+    enabled: options.enableMetrics,
+    port: options.metricsPort,
+    logger,
+  });
+
   const registryFilePath = path.join(options.home, registryFile);
   const { chains } = loadAndValidateRegistry(registryFilePath);
   const srcChain = chains[options.src];
@@ -224,10 +251,15 @@ async function run(options: Options, logger: Logger) {
     logger
   );
 
-  await relayerLoop(link, options, logger);
+  await relayerLoop(link, options, logger, metrics);
 }
 
-async function relayerLoop(link: Link, options: Options, logger: Logger) {
+async function relayerLoop(
+  link: Link,
+  options: Options,
+  logger: Logger,
+  metrics: Metrics
+) {
   let nextRelay = options.heights ?? {};
   const lastQueriedHeightsFilePath = path.join(
     options.home,
@@ -262,5 +294,7 @@ async function relayerLoop(link: Link, options: Options, logger: Logger) {
     logger.info(`Sleeping ${options.poll} seconds...`);
     await sleep(options.poll * 1000);
     logger.info('... waking up and checking for packets!');
+
+    metrics?.pollCounter?.inc();
   }
 }

src/binary/ibc-relayer/index.ts

@@ -38,6 +38,13 @@ const startCommand = program
   .addOption(mnemonicOption)
   .addOption(srcConnection)
   .addOption(destConnection)
+  .option(
+    '--enable-metrics',
+    'Enable Prometheus metrics collection and GET /metrics endpoint'
+  )
+  .option(
+    `--metrics-port <port>', 'Specify port for GET /metrics http server (default: ${startDefaults.metricsPort})`
+  )
   .option(
     '--poll <frequency>',
     `How many seconds we sleep between checking for packets (default: ${startDefaults.poll})`

src/binary/ibc-relayer/setup-prometheus.ts

@@ -0,0 +1,60 @@
+import http from 'http';
+
+import client from 'prom-client';
+
+import { Logger } from '../create-logger';
+
+let initialized = false;
+
+function getMetrics() {
+  return {
+    pollCounter: new client.Counter({
+      name: 'poll_counter',
+      help: 'Poll counter',
+    }),
+  };
+}
+
+export type Metrics = {
+  pollCounter: client.Counter<string>;
+} | null;
+export function setupPrometheus({
+  enabled,
+  port,
+  logger,
+}: {
+  enabled: boolean;
+  port: number;
+  logger: Logger;
+}): Metrics {
+  if (initialized) {
+    throw new Error(
+      `"setupPrometheus" func shouldn't be initialized more than once.`
+    );
+  }
+  initialized = true;
+
+  if (!enabled) {
+    return null;
+  }
+
+  client.collectDefaultMetrics({ prefix: 'confio_relayer_' });
+  const server = http.createServer(async (request, response) => {
+    if (request.method === 'GET' && request.url === '/metrics') {
+      const metrics = await client.register.metrics();
+      response.writeHead(200, {
+        'Content-Type': client.register.contentType,
+      });
+      response.end(metrics);
+
+      return;
+    }
+
+    response.writeHead(404);
+    response.end('404');
+  });
+  server.listen(port);
+  logger.info(`Prometheus GET /metrics exposed on port ${port}.`);
+
+  return getMetrics();
+}

src/binary/types.ts

@@ -22,6 +22,8 @@ export type AppConfig = {
   destConnection?: string;
   mnemonic?: string;
   keyFile?: string;
+  enableMetrics?: boolean;
+  metricsPort?: number;
 };
 
 export type LoggerFlags = {

src/binary/utils/load-and-validate-app.ts

@@ -41,6 +41,8 @@ export function loadAndValidateApp(home: string) {
       destConnection: { type: 'string', nullable: true },
       mnemonic: { type: 'string', nullable: true },
       keyFile: { type: 'string', nullable: true },
+      enableMetrics: { type: 'boolean', nullable: true },
+      metricsPort: { type: 'number', nullable: true },
     },
   };
   const validate = ajv.compile(schema);