[93] fix: Connected Grafana Api to the response-time-interceptor

packages/common/package.json

@@ -35,13 +35,14 @@
   "dependencies": {
     "@nestjs/cache-manager": "^2.1.0",
     "@nestjs/common": "^10.0.0",
-    "@nestjs/config": "^3.2.2",
+    "@nestjs/config": "^3.2.3",
     "@nestjs/core": "^10.0.0",
     "@nestjs/platform-express": "^10.0.0",
     "@nestjs/platform-fastify": "^10.3.0",
     "@temporalio/client": "^1.8.6",
     "@temporalio/worker": "^1.8.6",
     "@types/multer": "^1.4.11",
+    "axios": "^1.7.2",
     "cache-manager": "^5.2.4",
     "cache-manager-redis-store": "2",
     "fastify": "^4.27.0",

packages/common/src/controllers/prometheus.controller.ts

@@ -1,11 +1,12 @@
 import { Controller, Get, Res } from '@nestjs/common';
 import { FastifyReply } from 'fastify';
 import { register } from 'prom-client';
+
 @Controller()
 export class PrometheusController {
   @Get('metrics')
   async metrics(@Res() response: FastifyReply) {
-    response.headers({ 'Content-Type': register.contentType });
+    response.header('Content-Type', register.contentType);
     response.send(await register.metrics());
   }
 }

packages/common/src/interceptors/mutex.ts

@@ -0,0 +1,14 @@
+export class Mutex {
+    private mutex: Promise<void> = Promise.resolve();
+
+    lock(): PromiseLike<() => void> {
+      let begin: (unlock: () => void) => void = (unlock) => {};
+
+      this.mutex = this.mutex.then(() => new Promise(begin));
+
+      return new Promise((res) => {
+        begin = res;
+      });
+    }
+  }
+

packages/common/src/interceptors/response-time.interceptor.ts

@@ -1,64 +1,133 @@
-import {
-  Injectable,
-  NestInterceptor,
-  ExecutionContext,
-  CallHandler,
-  Logger,
-} from '@nestjs/common';
+import { Injectable, NestInterceptor, ExecutionContext, CallHandler, Logger } from '@nestjs/common';
 import { Observable, throwError } from 'rxjs';
 import { catchError, tap } from 'rxjs/operators';
 import { Histogram, exponentialBuckets } from 'prom-client';
-import * as fs from 'fs';
-import { generateBaseJSON, generateRow } from './utils';
+import axios from 'axios';
+import { Mutex } from './mutex';
+
+import {
+  generateBaseJSON,
+  generateRow,
+  getDashboardByUID,
+  getDashboardJSON,
+} from './utils';
 
+const mutex = new Mutex();
 @Injectable()
 export class ResponseTimeInterceptor implements NestInterceptor {
   private histogram: Histogram;
   private logger: Logger;
+  private dashboardUid: string;
+  private grafanaBaseURL: string;
+  private apiToken: string;
 
-  constructor(histogramTitle: string, jsonPath: string) {
+  constructor(
+    histogramTitle: string,
+    grafanaBaseURL: string,
+    apiToken: string,
+  ) {
     const name = histogramTitle + '_response_time';
     this.logger = new Logger(name + '_interceptor');
+    this.dashboardUid = null;
+    this.grafanaBaseURL = grafanaBaseURL;
+    this.apiToken = apiToken;
+    this.init(histogramTitle);
+  }
 
+  async init(histogramTitle: string) {
+    const name = histogramTitle + '_response_time';
     this.histogram = new Histogram({
       name: name,
       help: 'Response time of APIs',
       buckets: exponentialBuckets(1, 1.5, 30),
       labelNames: ['statusCode', 'endpoint'],
     });
 
-    // updating the grafana JSON with the row for this panel
+    const unlock = await mutex.lock(); 
     try {
-      // check if the path exists or not?
-      if (!fs.existsSync(jsonPath)) {
-        fs.writeFileSync(jsonPath, JSON.stringify(generateBaseJSON()), 'utf8'); // create file if not exists
-      }
+      const dashboardJSONSearchResp = await getDashboardJSON(
+        this.apiToken,
+        'Response_Times',
+        this.grafanaBaseURL,
+      );
+      this.dashboardUid =
+        dashboardJSONSearchResp.length > 0
+          ? dashboardJSONSearchResp[0]['uid']
+          : undefined;
+      let parsedContent: any;
 
-      const parsedContent = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
-
-      // skip creating the row if it already exists -- prevents multiple panels/rows on app restarts
-      let isPresent = false;
-      parsedContent.panels.forEach((panel: any) => {
-        // TODO: Make this verbose and add types -- convert the grafana JSON to TS Types/interface
-        if (
-          panel.title.trim() ===
-          name
-            .split('_')
-            .map((str) => str.charAt(0).toUpperCase() + str.slice(1))
-            .join(' ')
-            .trim()
-        ) {
-          isPresent = true;
+      if (this.dashboardUid === undefined || !this.dashboardUid) {
+        parsedContent = generateBaseJSON();
+
+        if (!parsedContent.dashboard.panels) {
+          parsedContent.dashboard.panels = [];
         }
-      });
-      if (isPresent) return;
 
-      parsedContent.panels.push(generateRow(name));
-      // write back to file
-      fs.writeFileSync(jsonPath, JSON.stringify(parsedContent));
+        if (!this.isPanelPresent(parsedContent.dashboard.panels, name)) {
+          parsedContent.dashboard.panels.push(generateRow(name));
+        }
+
+        const FINAL_JSON = parsedContent;
+        await axios.post(`${this.grafanaBaseURL}/api/dashboards/db`, FINAL_JSON, {
+          headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiToken}`
+          }
+        });
+      } else {
+        parsedContent = await getDashboardByUID(this.dashboardUid, this.grafanaBaseURL, this.apiToken);
+
+        if (!parsedContent.dashboard.panels) {
+          parsedContent.dashboard.panels = [];
+        }
+
+        if (!this.isPanelPresent(parsedContent.dashboard.panels, name)) {
+          parsedContent.dashboard.panels.push(generateRow(name));
+        }
+        await this.updateDashboard(parsedContent);
+      }
       this.logger.log('Successfully added histogram to dashboard!');
     } catch (err) {
-      this.logger.error('Error updating grafana JSON!', err);
+      console.error('Error updating Grafana JSON!');
+    } finally {
+      unlock();
+    }
+  }
+  isPanelPresent(panels: any[], name: string): boolean {
+    const formattedName = name
+      .split('_')
+      .map((str) => str.charAt(0).toUpperCase() + str.slice(1))
+      .join(' ')
+      .trim();
+
+    return panels.some((panel) => panel.title.trim() === formattedName);
+  }
+
+  async updateDashboard(FINAL_JSON: any) {
+    try {
+      await axios.post(`${this.grafanaBaseURL}/api/dashboards/db`, FINAL_JSON, {
+        headers: {
+          'Accept': 'application/json',
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiToken}`
+        }
+      });
+    } catch (err) {
+      const updatedDashboard = FINAL_JSON;
+      if (err.response && err.response.data && err.response.data.status === 'version-mismatch') {
+        this.logger.log('Dashboard version mismatch, retrying with latest version...');
+        updatedDashboard.dashboard.version++;
+        await axios.post(`${this.grafanaBaseURL}/api/dashboards/db`, updatedDashboard, {
+          headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiToken}`
+          }
+        });
+      } else {
+        throw err;
+      }
     }
   }
 
@@ -70,26 +139,24 @@ export class ResponseTimeInterceptor implements NestInterceptor {
     const startTime = performance.now();
     return next.handle().pipe(
       tap(() => {
-        // handles when there is no error propagating from the services to the controller
         const endTime = performance.now();
         const responseTime = endTime - startTime;
         const statusCode = response.statusCode;
         const endpoint = request.url;
         this.histogram.labels({ statusCode, endpoint }).observe(responseTime);
       }),
       catchError((err) => {
-        // handles when an exception is to be returned to the client
-        this.logger.error('error: ', err);
         const endTime = performance.now();
         const responseTime = endTime - startTime;
         const endpoint = request.url;
-        this.histogram
-          .labels({ statusCode: err.status, endpoint })
-          .observe(responseTime);
+        this.histogram.labels({ statusCode: err.status, endpoint }).observe(responseTime);
         return throwError(() => {
           throw err;
         });
       }),
     );
   }
 }
+
+
+