From ffc764b04e55e22bbedcf9f3389f321ce88a7b47 Mon Sep 17 00:00:00 2001 From: Yaroslav Grishajev Date: Wed, 18 Sep 2024 11:16:30 +0200 Subject: [PATCH] feat(logging): implement fluentd reporter closes #370 --- apps/api/package.json | 3 +- apps/api/src/core/config/env.config.ts | 6 +- .../src/core/providers/postgres.provider.ts | 2 +- .../core/services/logger/logger.service.ts | 55 ++++++++++-- .../postgres-logger.service.ts | 12 ++- apps/api/src/db/dbConnection.ts | 3 +- apps/api/src/types/pino-fluentd.d.ts | 12 +++ apps/api/webpack.dev.js | 2 +- apps/api/webpack.prod.js | 2 +- package-lock.json | 89 +++++++++++++++++-- 10 files changed, 166 insertions(+), 20 deletions(-) create mode 100644 apps/api/src/types/pino-fluentd.d.ts diff --git a/apps/api/package.json b/apps/api/package.json index 1fd75e3b3..80822cf2e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -80,7 +80,8 @@ "node-fetch": "^2.6.1", "pg": "^8.12.0", "pg-hstore": "^2.3.4", - "pino": "^9.2.0", + "pino": "^9.4.0", + "pino-fluentd": "^0.2.4", "pino-pretty": "^11.2.1", "postgres": "^3.4.4", "protobufjs": "^6.11.2", diff --git a/apps/api/src/core/config/env.config.ts b/apps/api/src/core/config/env.config.ts index ebe9503c5..bd10bdf20 100644 --- a/apps/api/src/core/config/env.config.ts +++ b/apps/api/src/core/config/env.config.ts @@ -2,8 +2,12 @@ import { z } from "zod"; const envSchema = z.object({ LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"), + STD_OUT_LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"), + SQL_LOG_FORMAT: z.enum(["raw", "pretty"]).optional().default("raw"), + FLUENTD_TAG: z.string().optional().default("pino"), + FLUENTD_HOST: z.string().optional(), + FLUENTD_PORT: z.number({ coerce: true }).optional().default(24224), NODE_ENV: z.enum(["development", "production", "test"]).optional().default("development"), - LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"), // TODO: make required once billing is in prod POSTGRES_DB_URI: z.string().optional(), POSTGRES_MAX_CONNECTIONS: z.number({ coerce: true }).optional().default(20), diff --git a/apps/api/src/core/providers/postgres.provider.ts b/apps/api/src/core/providers/postgres.provider.ts index 225df04ac..b3aa3f41e 100644 --- a/apps/api/src/core/providers/postgres.provider.ts +++ b/apps/api/src/core/providers/postgres.provider.ts @@ -15,7 +15,7 @@ const migrationClient = postgres(config.POSTGRES_DB_URI, { max: 1, onnotice: log const appClient = postgres(config.POSTGRES_DB_URI, { max: config.POSTGRES_MAX_CONNECTIONS, onnotice: logger.info.bind(logger) }); const schema = { ...userSchemas, ...billingSchemas }; -const drizzleOptions = { logger: new DefaultLogger({ writer: new PostgresLoggerService() }), schema }; +const drizzleOptions = { logger: new DefaultLogger({ writer: new PostgresLoggerService({ useFormat: config.SQL_LOG_FORMAT === "pretty" }) }), schema }; const pgMigrationDatabase = drizzle(migrationClient, drizzleOptions); export const migratePG = () => migrate(pgMigrationDatabase, { migrationsFolder: config.DRIZZLE_MIGRATIONS_FOLDER }); diff --git a/apps/api/src/core/services/logger/logger.service.ts b/apps/api/src/core/services/logger/logger.service.ts index f9468cb46..f0905461f 100644 --- a/apps/api/src/core/services/logger/logger.service.ts +++ b/apps/api/src/core/services/logger/logger.service.ts @@ -1,22 +1,65 @@ import { isHttpError } from "http-errors"; -import pino, { Bindings, LoggerOptions } from "pino"; +import pino, { Bindings } from "pino"; +import pinoFluentd from "pino-fluentd"; import pretty from "pino-pretty"; +import { Writable } from "stream"; import { config } from "@src/core/config"; export class LoggerService { protected pino: pino.Logger; - readonly isPretty = config.LOG_FORMAT === "pretty"; - constructor(bindings?: Bindings) { - const options: LoggerOptions = { level: config.LOG_LEVEL }; + this.pino = this.initPino(bindings); + } + + private initPino(bindings?: Bindings): pino.Logger { + const destinations: Writable[] = []; + + if (config.STD_OUT_LOG_FORMAT === "pretty") { + destinations.push(pretty({ sync: true })); + } else { + destinations.push(process.stdout); + } + + const fluentd = this.initFluentd(); + + if (fluentd) { + destinations.push(fluentd); + } - this.pino = pino(options, config.NODE_ENV === "production" ? undefined : pretty({ sync: true })); + let instance = pino({ level: config.LOG_LEVEL }, this.combineDestinations(destinations)); if (bindings) { - this.pino = this.pino.child(bindings); + instance = instance.child(bindings); } + + return instance; + } + + private initFluentd(): Writable | undefined { + const isFluentdEnabled = !!(config.FLUENTD_HOST && config.FLUENTD_PORT && config.FLUENTD_TAG); + + if (isFluentdEnabled) { + return pinoFluentd({ + tag: config.FLUENTD_TAG, + host: config.FLUENTD_HOST, + port: config.FLUENTD_PORT, + "trace-level": config.LOG_LEVEL + }); + } + } + + private combineDestinations(destinations: Writable[]): Writable { + return new Writable({ + write(chunk, encoding, callback) { + for (const destination of destinations) { + destination.write(chunk, encoding); + } + + callback(); + } + }); } info(message: any) { diff --git a/apps/api/src/core/services/postgres-logger/postgres-logger.service.ts b/apps/api/src/core/services/postgres-logger/postgres-logger.service.ts index e1b0680b4..0e48f778e 100644 --- a/apps/api/src/core/services/postgres-logger/postgres-logger.service.ts +++ b/apps/api/src/core/services/postgres-logger/postgres-logger.service.ts @@ -3,21 +3,29 @@ import { format } from "sql-formatter"; import { LoggerService } from "@src/core/services/logger/logger.service"; +interface PostgresLoggerServiceOptions { + orm?: "drizzle" | "sequelize"; + useFormat?: boolean; +} + export class PostgresLoggerService implements LogWriter { private readonly logger: LoggerService; private readonly isDrizzle: boolean; - constructor(options?: { orm: "drizzle" | "sequelize" }) { + private readonly useFormat: boolean; + + constructor(options?: PostgresLoggerServiceOptions) { const orm = options?.orm || "drizzle"; this.logger = new LoggerService({ context: "POSTGRES", orm }); this.isDrizzle = orm === "drizzle"; + this.useFormat = options?.useFormat || false; } write(message: string) { let formatted = message.replace(this.isDrizzle ? /^Query: / : /^Executing \(default\):/, ""); - if (this.logger.isPretty) { + if (this.useFormat) { formatted = format(formatted, { language: "postgresql" }); } diff --git a/apps/api/src/db/dbConnection.ts b/apps/api/src/db/dbConnection.ts index 242f649c8..be9581578 100644 --- a/apps/api/src/db/dbConnection.ts +++ b/apps/api/src/db/dbConnection.ts @@ -5,6 +5,7 @@ import pg from "pg"; import { Transaction as DbTransaction } from "sequelize"; import { Sequelize } from "sequelize-typescript"; +import { config } from "@src/core/config"; import { PostgresLoggerService } from "@src/core/services/postgres-logger/postgres-logger.service"; import { env } from "@src/utils/env"; @@ -26,7 +27,7 @@ if (!csMap[env.NETWORK]) { throw new Error(`Missing connection string for network: ${env.NETWORK}`); } -const logger = new PostgresLoggerService({ orm: "sequelize" }); +const logger = new PostgresLoggerService({ orm: "sequelize", useFormat: config.SQL_LOG_FORMAT === "pretty" }); const logging = (msg: string) => logger.write(msg); pg.defaults.parseInt8 = true; diff --git a/apps/api/src/types/pino-fluentd.d.ts b/apps/api/src/types/pino-fluentd.d.ts new file mode 100644 index 000000000..6cddba1b1 --- /dev/null +++ b/apps/api/src/types/pino-fluentd.d.ts @@ -0,0 +1,12 @@ +declare module "pino-fluentd" { + import { Writable } from "stream"; + + interface PinoFluentdOptions { + tag: string; + host: string; + port: number; + "trace-level": string; + } + + export default function pinoFluentd(options: PinoFluentdOptions): Writable; +} diff --git a/apps/api/webpack.dev.js b/apps/api/webpack.dev.js index 4393114b1..8fac94e89 100644 --- a/apps/api/webpack.dev.js +++ b/apps/api/webpack.dev.js @@ -19,7 +19,7 @@ module.exports = { extensions: [".ts", ".js"], alias: hq.get("webpack") }, - externals: [nodeExternals()], + externals: [nodeExternals(), { "winston-transport": "commonjs winston-transport" }], module: { rules: [ { diff --git a/apps/api/webpack.prod.js b/apps/api/webpack.prod.js index 87cddcff3..657238fda 100644 --- a/apps/api/webpack.prod.js +++ b/apps/api/webpack.prod.js @@ -18,7 +18,7 @@ module.exports = { extensions: [".ts", ".js"], alias: hq.get("webpack") }, - externals: [nodeExternals()], + externals: [nodeExternals(), { "winston-transport": "commonjs winston-transport" }], module: { rules: [ { diff --git a/package-lock.json b/package-lock.json index 4045897ad..2e4c2c796 100644 --- a/package-lock.json +++ b/package-lock.json @@ -80,7 +80,8 @@ "node-fetch": "^2.6.1", "pg": "^8.12.0", "pg-hstore": "^2.3.4", - "pino": "^9.2.0", + "pino": "^9.4.0", + "pino-fluentd": "^0.2.4", "pino-pretty": "^11.2.1", "postgres": "^3.4.4", "protobufjs": "^6.11.2", @@ -180,15 +181,16 @@ } }, "apps/api/node_modules/pino": { - "version": "9.2.0", - "license": "MIT", + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.4.0.tgz", + "integrity": "sha512-nbkQb5+9YPhQRz/BeQmrWpEknAaqjpAqRK8NwJpmrX/JHu7JuZC5G1CeAwJDJfGes4h+YihC6in3Q2nGb+Y09w==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^1.2.0", "pino-std-serializers": "^7.0.0", - "process-warning": "^3.0.0", + "process-warning": "^4.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", @@ -212,8 +214,9 @@ "license": "MIT" }, "apps/api/node_modules/process-warning": { - "version": "3.0.0", - "license": "MIT" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.0.tgz", + "integrity": "sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw==" }, "apps/api/node_modules/readable-stream": { "version": "4.5.2", @@ -19890,6 +19893,11 @@ "fast-safe-stringify": "^2.0.6" } }, + "node_modules/event-lite": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/event-lite/-/event-lite-0.1.3.tgz", + "integrity": "sha512-8qz9nOz5VeD2z96elrEKD2U433+L3DWdUdDkOINLGOJvx1GsMBbMn0aCeu28y8/e85A6mCigBiFlYMnTBEGlSw==" + }, "node_modules/event-target-shim": { "version": "5.0.1", "license": "MIT", @@ -20215,6 +20223,11 @@ "node": ">= 6" } }, + "node_modules/fast-json-parse": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fast-json-parse/-/fast-json-parse-1.0.3.tgz", + "integrity": "sha512-FRWsaZRWEJ1ESVNbDWmsAlqDk96gPQezzLghafp5J4GUKjbCz3OkAHuZs5TuPEtkbVQERysLp9xv6c24fBm8Aw==" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "license": "MIT" @@ -20531,6 +20544,17 @@ "node": ">=0.4.0" } }, + "node_modules/fluent-logger": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/fluent-logger/-/fluent-logger-3.4.1.tgz", + "integrity": "sha512-lERIhXAvhtCYeQq8K7sBDg/HY9GkiVRq5xY3oN+hcSINVKwqwBzG6LQOJK73EnV50qO59U7XEmRnn2hBzLWaHw==", + "dependencies": { + "msgpack-lite": "*" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/follow-redirects": { "version": "1.15.6", "funding": [ @@ -22004,6 +22028,11 @@ "dev": true, "license": "0BSD" }, + "node_modules/int64-buffer": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.10.tgz", + "integrity": "sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==" + }, "node_modules/internal-slot": { "version": "1.0.7", "license": "MIT", @@ -27717,6 +27746,25 @@ "version": "2.1.2", "license": "MIT" }, + "node_modules/msgpack-lite": { + "version": "0.1.26", + "resolved": "https://registry.npmjs.org/msgpack-lite/-/msgpack-lite-0.1.26.tgz", + "integrity": "sha512-SZ2IxeqZ1oRFGo0xFGbvBJWMp3yLIY9rlIJyxy8CGrwZn1f0ZK4r6jV/AM1r0FZMDUkWkglOk/eeKIL9g77Nxw==", + "dependencies": { + "event-lite": "^0.1.1", + "ieee754": "^1.1.8", + "int64-buffer": "^0.1.9", + "isarray": "^1.0.0" + }, + "bin": { + "msgpack": "bin/msgpack" + } + }, + "node_modules/msgpack-lite/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/multiformats": { "version": "9.9.0", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", @@ -29345,6 +29393,35 @@ "split2": "^4.0.0" } }, + "node_modules/pino-fluentd": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/pino-fluentd/-/pino-fluentd-0.2.4.tgz", + "integrity": "sha512-5f4Vssz2CiuakOj1J5upqPz7fY2mM2h7HA6sxTaM4FGTXxu6/4IvpeV8B97WEqh3frct1vBC0i+smfFKzrLEQA==", + "dependencies": { + "fast-json-parse": "^1.0.3", + "fluent-logger": "^3.4.1", + "minimist": "^1.2.5", + "pump": "^3.0.0", + "readable-stream": "^3.6.0", + "split2": "^4.1.0" + }, + "bin": { + "pino-fluentd": "pino-fluentd.js" + } + }, + "node_modules/pino-fluentd/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/pino-pretty": { "version": "11.2.1", "license": "MIT",