modified: README.md

modified: apps/api/config/config.devnet.yaml modified: apps/api/src/endpoints/hello-challenge/hello-challenge.controller.ts modified: apps/api/src/main.ts modified: apps/transactions-processor/config/config.devnet.yaml modified: apps/transactions-processor/src/processor/transaction.processor.service.ts new file: docker-compose.dev.yml modified: helm/mx-challenge-app/dashboards/transactions.json new file: helm/mx-challenge-app/dashboards/transactions_v2.json modified: helm/mx-challenge-app/mx-challenge-app-0.1.0.tgz modified: helm/mx-challenge-app/templates/api/deployment.yaml modified: helm/mx-challenge-app/templates/api/service.yaml modified: helm/mx-challenge-app/templates/grafana-dashboards-configmap.yaml modified: helm/mx-challenge-app/values.yaml modified: libs/common/src/config/api.config.service.ts modified: libs/common/src/metrics/api.metrics.service.ts modified: libs/common/src/pubsub/pub.sub.listener.controller.ts modified: libs/common/src/utils/transactions.event.dto.ts new file: monitoring/grafana/dashboards/transactions v2.json modified: package-lock.json modified: package.json
mradian1 · May 28, 2024 · f5bb7c3 · f5bb7c3
1 parent fbb6102
commit f5bb7c3
Show file tree

Hide file tree

Showing 21 changed files with 1,373 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -2,13 +2,23 @@
 
 ## Changes made
 
+v1
+
 1. Transaction-processor: onTransactionReceived emits an event that communicates the shard_id and transactions value
 2. PubSub: added event route for transactions, that passes the values received to the ApiMetrics
 3. ApiMetrics: added a transactions gauge and its set function. getMetrics will reset the gauge in order not to persist old data when no transactions are received for that shard.
 4. Config: added host settings for mongodb and respective get function in order to be able to set it independently in deployment mode.
    Also added the possibility to read all relevant config values from env variables, as they are passed as such in deployment mode
 5. Deployment (work in progress) executed through helm charts. Grafana doesn't, as yet, load json file for predefined dashboard, but data can be visualised by creating a new dashboard.
 
+v2
+6. Added api url in swagger url options in order to be able to test api routes
+7. Modified TransactionProcessor:
+   a. value is now expressed in egld
+   b. cross shard transactions (sourceShard is not current shard) are ignored in order to avoid doubling the data.
+   c. transaction processor is imported from the mradian1/sdk-transaction-processor repository - branch timestamp, which now provides timestamp and round as parameters to onTransactionReceived
+8. Modified getMetrics - clearGauge() function resets gauge values to 0, instead of reset() which resets Gauge values to undefined
+
 ## Quick start
 
 1. Run `npm install` in the project directory

diff --git a/apps/api/config/config.devnet.yaml b/apps/api/config/config.devnet.yaml
@@ -4,11 +4,11 @@ urls:
     - 'https://devnet-microservice.multiversx.com'
     - 'https://testnet-microservice.multiversx.com'
     - 'https://microservice.multiversx.com'
-  redis: '127.0.0.1' #'redis'
+  redis: 'redis'
 nosql:
-  host: 'localhost' #'mongodb'
+  host: 'mongodb'
 database:
-  host: 'localhost' #'database'
+  host: 'database'
   port: 3306
   username: 'root'
   password: 'root'

diff --git a/apps/api/src/endpoints/hello-challenge/hello-challenge.controller.ts b/apps/api/src/endpoints/hello-challenge/hello-challenge.controller.ts
@@ -1,5 +1,6 @@
 import { Controller, Get } from '@nestjs/common';
-
+import { ApiTags } from '@nestjs/swagger';
+@ApiTags('challenge')
 @Controller('challenge')
 export class HelloChallengeController {
   @Get()

diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
@@ -72,6 +72,7 @@ async function bootstrap() {
   for (const apiUrl of apiUrls) {
     documentBuilder = documentBuilder.addServer(apiUrl);
   }
+  documentBuilder.addServer(apiConfigService.getLocalSwaggerUrl());
 
   const config = documentBuilder.build();
 

diff --git a/apps/transactions-processor/config/config.devnet.yaml b/apps/transactions-processor/config/config.devnet.yaml
@@ -1,6 +1,6 @@
 urls:
   api: 'https://devnet-api.multiversx.com'
-  redis: '127.0.0.1'
+  redis: 'redis' #'127.0.0.1'
 features:
   transactionProcessor:
     port: 5202

diff --git a/apps/transactions-processor/src/processor/transaction.processor.service.ts b/apps/transactions-processor/src/processor/transaction.processor.service.ts
@@ -1,14 +1,11 @@
 import { CacheService } from '@multiversx/sdk-nestjs-cache';
 import { Locker } from '@multiversx/sdk-nestjs-common';
-import { TransactionProcessor } from '@multiversx/sdk-transaction-processor';
+//import { TransactionProcessor } from '@multiversx/sdk-transaction-processor';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { Cron } from '@nestjs/schedule';
-import {
-  ApiConfigService,
-  //ApiMetricsService,
-  CacheInfo,
-} from '@mvx-monorepo/common';
+import { ApiConfigService, CacheInfo } from '@mvx-monorepo/common';
 import { ClientProxy } from '@nestjs/microservices';
+import { TransactionProcessor } from '@mradian1/sdk-transaction-processor-branch';
 
 @Injectable()
 export class TransactionProcessorService {
@@ -36,20 +33,25 @@ export class TransactionProcessorService {
         onTransactionsReceived: async (
           shardId,
           nonce,
+          round,
+          timestamp,
           transactions,
           statistics,
+          _blockHash,
         ) => {
-          const totalValue: number = transactions
-            .map((transaction) => parseInt(transaction.value))
+          const totalValue: bigint = transactions
+            .map((transaction) => transaction.sourceShard == shardId ?BigInt(transaction.value) : BigInt(0))
             .reduce((a, b) => a + b);
+
           this.logger.log(
-            `Received ${transactions.length} transactions of value ${totalValue} on shard ${shardId} and nonce ${nonce}. Time left: ${statistics.secondsLeft}`,
+            `Received ${transactions.length} transactions of value ${totalValue} = ${this.toEgld(totalValue,8)}egld on shard ${shardId} and nonce ${nonce}. Time stamp: ${timestamp} - round: ${round}, time left: ${statistics.secondsLeft} sec`,
           );
-          //await this.metricsService.setTransactionsValue(shardId, totalValue);
+
           this.client.emit('onTransactions', {
             length: transactions.length,
-            value: totalValue,
+            value: this.toEgld(totalValue,8), //totalValue,
             shardId: shardId,
+            timestamp: timestamp,
           });
         },
         getLastProcessedNonce: async (shardId) => {
@@ -67,4 +69,10 @@ export class TransactionProcessorService {
       });
     });
   }
+  toEgld(bigIntVal: bigint, precision: number): number {
+    const divisor = 10**precision;
+    const convertToEgld: bigint = BigInt (10 ** 18);
+    return Number((bigIntVal*BigInt(divisor)/convertToEgld))/divisor;
+  }
 }
+
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -0,0 +1,109 @@
+version: '3.4'
+
+services:
+  api:
+    build:
+      context: .
+      dockerfile: ./apps/api/Dockerfile
+      target: development
+    command: npm run start:devnet:watch
+    depends_on:
+      - mongodb
+      - database
+      - redis
+    volumes:
+      - .:/usr/src/app
+      #- /usr/src/app/node_modules
+    ports:
+      - '3000:3000'
+      - '3005:3005'
+      - '4000:4000'
+  cache-warmer:
+    build:
+      context: .
+      dockerfile: ./apps/cache-warmer/Dockerfile
+      target: development
+    command: npm run start:cache-warmer:devnet:watch
+    depends_on:
+      - mongodb
+      - database
+      - redis
+    volumes:
+      - .:/usr/src/app
+      #- /usr/src/app/node_modules
+    ports:
+      - '5201:5201'
+
+  queue-worker:
+    build:
+      context: .
+      dockerfile: ./apps/queue-worker/Dockerfile
+      target: development
+    command: npm run start:queue-worker:devnet:watch
+    depends_on:
+      - mongodb
+      - database
+      - redis
+    volumes:
+      - .:/usr/src/app
+      #- /usr/src/app/node_modules
+    ports:
+      - '8000:8000'
+
+  transactions-processor:
+    build:
+      context: .
+      dockerfile: ./apps/transactions-processor/Dockerfile
+      target: development
+    command: npm run start:transactions-processor:devnet:watch
+    depends_on:
+      - mongodb
+      - database
+      - redis
+    volumes:
+      - .:/usr/src/app
+      #- /usr/src/app/node_modules
+    ports:
+      - '5202:5202'
+
+  redis:
+    image: "redis:alpine"
+    command: redis-server
+    ports:
+      - "6379:6379"
+    environment:
+      - REDIS_REPLICATION_MODE=master
+
+  database:
+    image: "mysql:latest"
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=root
+      - MYSQL_DATABASE=example
+
+  mongodb:
+    image: mongo:latest
+    environment:
+      - MONGODB_DATABASE="example"
+    ports:
+      - 27017:27017
+
+  prometheus:
+    image: prom/prometheus:latest
+    command: '--config.file=/etc/prometheus/prometheus.yml'
+    volumes:
+      - ./monitoring/prometheus:/etc/prometheus
+    ports:
+      - 9090:9090
+    extra_hosts:
+      - host.docker.internal:host-gateway
+
+  grafana:
+    depends_on:
+      - prometheus
+    image: grafana/grafana:latest
+    volumes:
+      - ./monitoring/grafana:/etc/grafana/provisioning
+    ports:
+      - 3010:3000
diff --git a/helm/mx-challenge-app/dashboards/transactions.json b/helm/mx-challenge-app/dashboards/transactions.json
@@ -18,7 +18,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 1,
   "links": [],
   "panels": [
     {
@@ -38,7 +37,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "drawStyle": "bars",
+            "drawStyle": "line",
             "fillOpacity": 11,
             "gradientMode": "none",
             "hideFrom": {
@@ -47,7 +46,7 @@
               "viz": false
             },
             "insertNulls": false,
-            "lineInterpolation": "linear",
+            "lineInterpolation": "smooth",
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
@@ -115,6 +114,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -123,7 +123,8 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "transactions_total_value",
+          "expr": "transactions_total_value_timestamp",
+          "format": "table",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -135,6 +136,68 @@
       ],
       "title": "Panel Title",
       "transformations": [
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "dateFormat": "X",
+                "destinationType": "time",
+                "targetField": "timestamp"
+              }
+            ],
+            "fields": {}
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "app": true,
+              "instance": true,
+              "job": true,
+              "namespace": true,
+              "pod": true,
+              "pod_template_hash": true,
+              "shardId": false
+            },
+            "includeByName": {},
+            "indexByName": {
+              "Time": 0,
+              "Total": 11,
+              "Value": 10,
+              "__name__": 1,
+              "app": 2,
+              "instance": 3,
+              "job": 4,
+              "namespace": 5,
+              "pod": 6,
+              "pod_template_hash": 7,
+              "shardId": 9,
+              "timestamp": 8
+            },
+            "renameByName": {}
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "timestamp"
+              }
+            ]
+          }
+        },
+        {
+          "id": "prepareTimeSeries",
+          "options": {
+            "format": "multi"
+          }
+        },
         {
           "id": "calculateField",
           "options": {}
@@ -150,7 +213,7 @@
     "list": []
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-15m",
     "to": "now"
   },
   "timepicker": {},