feat(logging): implement fluentd reporter

closes #370

apps/api/package.json

@@ -80,7 +80,8 @@
     "node-fetch": "^2.6.1",
     "pg": "^8.12.0",
     "pg-hstore": "^2.3.4",
-    "pino": "^9.2.0",
+    "pino": "^9.4.0",
+    "pino-fluentd": "^0.2.4",
     "pino-pretty": "^11.2.1",
     "postgres": "^3.4.4",
     "protobufjs": "^6.11.2",

apps/api/src/core/config/env.config.ts

@@ -2,8 +2,12 @@ import { z } from "zod";
 
 const envSchema = z.object({
   LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"),
+  STD_OUT_LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"),
+  SQL_LOG_FORMAT: z.enum(["raw", "pretty"]).optional().default("raw"),
+  FLUENTD_TAG: z.string().optional().default("pino"),
+  FLUENTD_HOST: z.string().optional(),
+  FLUENTD_PORT: z.number({ coerce: true }).optional().default(24224),
   NODE_ENV: z.enum(["development", "production", "test"]).optional().default("development"),
-  LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"),
   // TODO: make required once billing is in prod
   POSTGRES_DB_URI: z.string().optional(),
   POSTGRES_MAX_CONNECTIONS: z.number({ coerce: true }).optional().default(20),

apps/api/src/core/providers/postgres.provider.ts

@@ -15,7 +15,7 @@ const migrationClient = postgres(config.POSTGRES_DB_URI, { max: 1, onnotice: log
 const appClient = postgres(config.POSTGRES_DB_URI, { max: config.POSTGRES_MAX_CONNECTIONS, onnotice: logger.info.bind(logger) });
 
 const schema = { ...userSchemas, ...billingSchemas };
-const drizzleOptions = { logger: new DefaultLogger({ writer: new PostgresLoggerService() }), schema };
+const drizzleOptions = { logger: new DefaultLogger({ writer: new PostgresLoggerService({ useFormat: config.SQL_LOG_FORMAT === "pretty" }) }), schema };
 
 const pgMigrationDatabase = drizzle(migrationClient, drizzleOptions);
 export const migratePG = () => migrate(pgMigrationDatabase, { migrationsFolder: config.DRIZZLE_MIGRATIONS_FOLDER });

apps/api/src/core/services/logger/logger.service.ts

@@ -1,22 +1,65 @@
 import { isHttpError } from "http-errors";
-import pino, { Bindings, LoggerOptions } from "pino";
+import pino, { Bindings } from "pino";
+import pinoFluentd from "pino-fluentd";
 import pretty from "pino-pretty";
+import { Writable } from "stream";
 
 import { config } from "@src/core/config";
 
 export class LoggerService {
   protected pino: pino.Logger;
 
-  readonly isPretty = config.LOG_FORMAT === "pretty";
-
   constructor(bindings?: Bindings) {
-    const options: LoggerOptions = { level: config.LOG_LEVEL };
+    this.pino = this.initPino(bindings);
+  }
+
+  private initPino(bindings?: Bindings): pino.Logger {
+    const destinations: Writable[] = [];
+
+    if (config.STD_OUT_LOG_FORMAT === "pretty") {
+      destinations.push(pretty({ sync: true }));
+    } else {
+      destinations.push(process.stdout);
+    }
+
+    const fluentd = this.initFluentd();
+
+    if (fluentd) {
+      destinations.push(fluentd);
+    }
 
-    this.pino = pino(options, config.NODE_ENV === "production" ? undefined : pretty({ sync: true }));
+    let instance = pino({ level: config.LOG_LEVEL }, this.combineDestinations(destinations));
 
     if (bindings) {
-      this.pino = this.pino.child(bindings);
+      instance = instance.child(bindings);
     }
+
+    return instance;
+  }
+
+  private initFluentd(): Writable | undefined {
+    const isFluentdEnabled = !!(config.FLUENTD_HOST && config.FLUENTD_PORT && config.FLUENTD_TAG);
+
+    if (isFluentdEnabled) {
+      return pinoFluentd({
+        tag: config.FLUENTD_TAG,
+        host: config.FLUENTD_HOST,
+        port: config.FLUENTD_PORT,
+        "trace-level": config.LOG_LEVEL
+      });
+    }
+  }
+
+  private combineDestinations(destinations: Writable[]): Writable {
+    return new Writable({
+      write(chunk, encoding, callback) {
+        for (const destination of destinations) {
+          destination.write(chunk, encoding);
+        }
+
+        callback();
+      }
+    });
   }
 
   info(message: any) {

apps/api/src/core/services/postgres-logger/postgres-logger.service.ts

@@ -3,21 +3,29 @@ import { format } from "sql-formatter";
 
 import { LoggerService } from "@src/core/services/logger/logger.service";
 
+interface PostgresLoggerServiceOptions {
+  orm?: "drizzle" | "sequelize";
+  useFormat?: boolean;
+}
+
 export class PostgresLoggerService implements LogWriter {
   private readonly logger: LoggerService;
 
   private readonly isDrizzle: boolean;
 
-  constructor(options?: { orm: "drizzle" | "sequelize" }) {
+  private readonly useFormat: boolean;
+
+  constructor(options?: PostgresLoggerServiceOptions) {
     const orm = options?.orm || "drizzle";
     this.logger = new LoggerService({ context: "POSTGRES", orm });
     this.isDrizzle = orm === "drizzle";
+    this.useFormat = options?.useFormat || false;
   }
 
   write(message: string) {
     let formatted = message.replace(this.isDrizzle ? /^Query: / : /^Executing \(default\):/, "");
 
-    if (this.logger.isPretty) {
+    if (this.useFormat) {
       formatted = format(formatted, { language: "postgresql" });
     }
 

apps/api/src/db/dbConnection.ts

@@ -5,6 +5,7 @@ import pg from "pg";
 import { Transaction as DbTransaction } from "sequelize";
 import { Sequelize } from "sequelize-typescript";
 
+import { config } from "@src/core/config";
 import { PostgresLoggerService } from "@src/core/services/postgres-logger/postgres-logger.service";
 import { env } from "@src/utils/env";
 
@@ -26,7 +27,7 @@ if (!csMap[env.NETWORK]) {
   throw new Error(`Missing connection string for network: ${env.NETWORK}`);
 }
 
-const logger = new PostgresLoggerService({ orm: "sequelize" });
+const logger = new PostgresLoggerService({ orm: "sequelize", useFormat: config.SQL_LOG_FORMAT === "pretty" });
 const logging = (msg: string) => logger.write(msg);
 
 pg.defaults.parseInt8 = true;

apps/api/src/types/pino-fluentd.d.ts

@@ -0,0 +1,12 @@
+declare module "pino-fluentd" {
+  import { Writable } from "stream";
+
+  interface PinoFluentdOptions {
+    tag: string;
+    host: string;
+    port: number;
+    "trace-level": string;
+  }
+
+  export default function pinoFluentd(options: PinoFluentdOptions): Writable;
+}

apps/api/webpack.dev.js

@@ -19,7 +19,7 @@ module.exports = {
     extensions: [".ts", ".js"],
     alias: hq.get("webpack")
   },
-  externals: [nodeExternals()],
+  externals: [nodeExternals(), { "winston-transport": "commonjs winston-transport" }],
   module: {
     rules: [
       {

apps/api/webpack.prod.js

@@ -18,7 +18,7 @@ module.exports = {
     extensions: [".ts", ".js"],
     alias: hq.get("webpack")
   },
-  externals: [nodeExternals()],
+  externals: [nodeExternals(), { "winston-transport": "commonjs winston-transport" }],
   module: {
     rules: [
       {