modified: .dockerignore

modified: README.md modified: apps/api/Dockerfile modified: apps/api/config/config.devnet.yaml modified: apps/api/config/config.mainnet.yaml modified: apps/api/config/config.testnet.yaml modified: apps/api/src/websockets/events.gateway.ts modified: apps/api/src/websockets/pub.sub.controller.ts modified: apps/cache-warmer/Dockerfile modified: apps/cache-warmer/config/config.mainnet.yaml modified: apps/queue-worker/Dockerfile modified: apps/queue-worker/config/config.mainnet.yaml modified: apps/transactions-processor/Dockerfile modified: apps/transactions-processor/config/config.devnet.yaml modified: apps/transactions-processor/config/config.mainnet.yaml deleted: apps/transactions-processor/src/processor/transaction.processor.metrics.ts modified: docker-compose.yml new file: helm/mx-app/.helmignore new file: helm/mx-app/Chart.yaml new file: helm/mx-app/grafana-dashboards-configmap.yaml new file: helm/mx-app/grafana-datasource-configmap.yaml new file: helm/mx-app/grafana-deployment.yaml new file: helm/mx-app/prometheus-configmap.yaml new file: helm/mx-app/prometheus-deployment.yaml new file: helm/mx-app/templates/NOTES.txt new file: helm/mx-app/templates/_helpers.tpl new file: helm/mx-app/templates/deployment.yaml new file: helm/mx-app/templates/hpa.yaml new file: helm/mx-app/templates/ingress.yaml new file: helm/mx-app/templates/service.yaml new file: helm/mx-app/templates/serviceaccount.yaml new file: helm/mx-app/templates/tests/test-connection.yaml new file: helm/mx-app/values.yaml new file: helm/mx-challenge-app/.helmignore new file: helm/mx-challenge-app/Chart.lock new file: helm/mx-challenge-app/Chart.yaml new file: helm/mx-challenge-app/charts/grafana-7.3.11.tgz new file: helm/mx-challenge-app/charts/mongodb-15.4.3.tgz new file: helm/mx-challenge-app/charts/mysql-10.2.4.tgz new file: helm/mx-challenge-app/charts/prometheus-15.18.0.tgz new file: helm/mx-challenge-app/charts/redis-19.3.2.tgz new file: helm/mx-challenge-app/dashboards/template_service_dashboard.json new file: helm/mx-challenge-app/dashboards/transactions.json new file: helm/mx-challenge-app/templates/api/deployment.yaml new file: helm/mx-challenge-app/templates/cache-warmer/deployment.yaml new file: helm/mx-challenge-app/templates/grafana-datasource-configmap.yaml new file: helm/mx-challenge-app/templates/queue-worker/deployment.yaml new file: helm/mx-challenge-app/templates/transactions-processor/deployment.yaml.old new file: helm/mx-challenge-app/values.yaml new file: helm/package.json modified: libs/common/src/config/api.config.service.ts modified: libs/common/src/metrics/api.metrics.service.ts modified: libs/common/src/pubsub/pub.sub.listener.controller.ts modified: monitoring/grafana/dashboards/template_service_dashboard.json new file: monitoring/grafana/dashboards/template_service_dashboard.json.old modified: monitoring/prometheus/prometheus.yml modified: package-lock.json modified: package.json
mradian1 · May 19, 2024 · 524509b · 524509b
1 parent 1c22d27
commit 524509b
Show file tree

Hide file tree

Showing 58 changed files with 8,445 additions and 3,501 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -5,7 +5,7 @@
 
 # Build dependencies
 dist.do
-node_modules
+#node_modules
 
 # Environment (contains sensitive data)
 #*.env

diff --git a/README.md b/README.md
@@ -1,5 +1,14 @@
   REST API facade template for microservices that interacts with the MultiversX blockchain.
 
+## Changes made
+
+1. Transaction-processor: onTransactionReceived emits an event that communicates the shard_id and transactions value
+2. PubSub: added event route for transactions, that passes the values received to the ApiMetrics
+3. ApiMetrics: added a transactions gauge and its set function. getMetrics will reset the gauge in order not to persist old data when no transactions are received for that shard.
+4. Config: added host settings for mongodb and respective get function in order to be able to set it independently in deployment mode.
+   Also added the possibility to read all relevant config values from env variables, as they are passed as such in deployment mode
+5. Deployment (work in progress) executed through helm charts. Grafana doesn't, as yet, load json file for predefined dashboard, but data can be visualised by creating a new dashboard.
+
 ## Quick start
 
 1. Run `npm install` in the project directory
@@ -106,7 +115,7 @@ $ npm run start:queue-worker:devnet
 $ npm run start:queue-worker:mainnet
 ```
 
-Requests can be made to http://localhost:3001 for the api. The app will reload when you'll make edits (if opened in watch mode). You will also see any lint errors in the console.
+Requests can be made to <http://localhost:3001> for the api. The app will reload when you'll make edits (if opened in watch mode). You will also see any lint errors in the console.
 
 ### `npm run test`
 

diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
@@ -29,6 +29,4 @@ EXPOSE 3000
 EXPOSE 3005
 EXPOSE 4000
 
-CMD ["node", "dist/apps/api/main"]
-
-
+CMD ["node", "dist/apps/api/src/main"]
diff --git a/apps/api/config/config.devnet.yaml b/apps/api/config/config.devnet.yaml
@@ -4,9 +4,11 @@ urls:
     - 'https://devnet-microservice.multiversx.com'
     - 'https://testnet-microservice.multiversx.com'
     - 'https://microservice.multiversx.com'
-  redis: 'redis' #'127.0.0.1'
+  redis: '127.0.0.1' #'redis'
+nosql:
+  host: 'localhost' #'mongodb'
 database:
-  host: 'database' #'localhost'
+  host: 'localhost' #'database'
   port: 3306
   username: 'root'
   password: 'root'

diff --git a/apps/api/config/config.mainnet.yaml b/apps/api/config/config.mainnet.yaml
@@ -4,9 +4,11 @@ urls:
     - 'https://microservice.multiversx.com'
     - 'https://devnet-microservice.multiversx.com'
     - 'https://testnet-microservice.multiversx.com'
-  redis: '127.0.0.1'
+  redis: 'redis' #'127.0.0.1'
+nosql:
+  host: 'mongodb' #'localhost' 
 database:
-  host: 'localhost'
+  host: 'database' #'localhost'
   port: 3306
   username: 'root'
   password: 'root'

diff --git a/apps/api/config/config.testnet.yaml b/apps/api/config/config.testnet.yaml
@@ -5,6 +5,8 @@ urls:
     - 'https://devnet-microservice.multiversx.com'
     - 'https://microservice.multiversx.com'
   redis: '127.0.0.1'
+nosql:
+  host: 'localhost'
 database:
   host: 'localhost'
   port: 3306

diff --git a/apps/api/src/websockets/events.gateway.ts b/apps/api/src/websockets/events.gateway.ts
@@ -18,8 +18,4 @@ export class EventsGateway {
     this.logger.log(`Received onTest event with payload '${JSON.stringify(payload)}'`);
     this.webSocketServer?.emit('test', payload);
   }
-
-  onTransactionsReceived(payload: unknown) {
-    this.logger.log(`Received onTransaction event with payload '${JSON.stringify(payload)}'`);
-  }
 }
diff --git a/apps/api/src/websockets/pub.sub.controller.ts b/apps/api/src/websockets/pub.sub.controller.ts
@@ -17,9 +17,4 @@ export class PubSubController {
     this.logger.log(`Notifying onTest with payload '${JSON.stringify(payload)}'`);
     this.eventsGateway.onTest(payload);
   }
-  @EventPattern('onTransactions')
-  onTransactionsReceived(payload: unknown) {
-    this.logger.log(`Notifying onTransactionsReceived with payload '${JSON.stringify(payload)}'`);
-    this.eventsGateway.onTransactionsReceived(payload);
-  }
 }
diff --git a/apps/cache-warmer/Dockerfile b/apps/cache-warmer/Dockerfile
@@ -27,4 +27,5 @@ COPY --from=development /usr/src/app/dist ./dist
 
 EXPOSE 5201
 
-CMD ["node", "dist/apps/cache-warmer/main"]
+CMD ["node", "dist/apps/cache-warmer/src/main"]
+#CMD ["npm", "run", "start:cache-warmer:mainnet"]
diff --git a/apps/cache-warmer/config/config.mainnet.yaml b/apps/cache-warmer/config/config.mainnet.yaml
@@ -1,10 +1,10 @@
 urls:
   api: 'https://api.multiversx.com'
-  redis: '127.0.0.1'
+  redis: 'redis' #'127.0.0.1'
 features:
   cacheWarmer:
     enabled: true
     port: 5201
   privateApi:
-    enabled: true
-    port: 4000
+    enabled: false
+    port: 4000
diff --git a/apps/queue-worker/Dockerfile b/apps/queue-worker/Dockerfile
@@ -27,4 +27,5 @@ COPY --from=development /usr/src/app/dist ./dist
 
 EXPOSE 8000
 
-CMD ["node", "dist/apps/queue-worker/main"]
+CMD ["node", "dist/apps/queue-worker/src/main"]
+#CMD ["npm", "run", "start:queue-worker:mainnet"]
diff --git a/apps/queue-worker/config/config.mainnet.yaml b/apps/queue-worker/config/config.mainnet.yaml
@@ -1,10 +1,10 @@
 urls:
   api: 'https://devnet-api.multiversx.com'
-  redis: '127.0.0.1'
+  redis: 'redis' #'127.0.0.1'
 features:
   queueWorker:
     enabled: true
     port: 8000
   privateApi:
-    enabled: true
+    enabled: false
     port: 4000
diff --git a/apps/transactions-processor/Dockerfile b/apps/transactions-processor/Dockerfile
@@ -27,4 +27,5 @@ COPY --from=development /usr/src/app/dist ./dist
 
 EXPOSE 5202
 
-CMD ["node", "dist/apps/transactions-processor/main"]
+CMD ["node", "dist/apps/transactions-processor/src/main"]
+#CMD ["npm", "run", "start:transactions-processor:mainnet"]
diff --git a/apps/transactions-processor/config/config.devnet.yaml b/apps/transactions-processor/config/config.devnet.yaml
@@ -1,6 +1,6 @@
 urls:
   api: 'https://devnet-api.multiversx.com'
-  redis: 'redis' #'127.0.0.1'
+  redis: '127.0.0.1'
 features:
   transactionProcessor:
     port: 5202

diff --git a/apps/transactions-processor/config/config.mainnet.yaml b/apps/transactions-processor/config/config.mainnet.yaml
@@ -1,10 +1,10 @@
 urls:
   api: 'https://api.multiversx.com'
-  redis: '127.0.0.1'
+  redis: 'redis' #'127.0.0.1'
 features:
   transactionProcessor:
     port: 5202
     maxLookBehind: 100
   privateApi:
-    enabled: true
-    port: 4000
+    enabled: false
+    port: 4000
diff --git a/apps/transactions-processor/src/processor/transaction.processor.metrics.ts b/apps/transactions-processor/src/processor/transaction.processor.metrics.ts
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,15 +5,15 @@ services:
     build:
       context: .
       dockerfile: ./apps/api/Dockerfile
-      target: development
-    command: npm run start:devnet watch
+      target: production
+    command: npm run start:mainnet
     depends_on:
       - mongodb
       - database
       - redis
     volumes:
       - .:/usr/src/app
-      - /usr/src/app/node_modules
+      #- /usr/src/app/node_modules
     ports:
       - '3000:3000'
       - '3005:3005'
@@ -22,47 +22,47 @@ services:
     build:
       context: .
       dockerfile: ./apps/cache-warmer/Dockerfile
-      target: development
-    command: npm run start:cache-warmer:devnet:watch
+      target: production
+    command: npm run start:cache-warmer:mainnet
     depends_on:
       - mongodb
       - database
       - redis
     volumes:
       - .:/usr/src/app
-      - /usr/src/app/node_modules
+      #- /usr/src/app/node_modules
     ports:
       - '5201:5201'
 
   queue-worker:
     build:
       context: .
       dockerfile: ./apps/queue-worker/Dockerfile
-      target: development
-    command: npm run start:queue-worker:devnet:watch
+      target: production
+    command: npm run start:queue-worker:mainnet
     depends_on:
       - mongodb
       - database
       - redis
     volumes:
       - .:/usr/src/app
-      - /usr/src/app/node_modules
+      #- /usr/src/app/node_modules
     ports:
       - '8000:8000'
 
   transactions-processor:
     build:
       context: .
       dockerfile: ./apps/transactions-processor/Dockerfile
-      target: development
-    command: npm run start:transactions-processor:devnet:watch
+      target: production
+    command: npm run start:transactions-processor:mainnet
     depends_on:
       - mongodb
       - database
       - redis
     volumes:
       - .:/usr/src/app
-      - /usr/src/app/node_modules
+      #- /usr/src/app/node_modules
     ports:
       - '5202:5202'
 

diff --git a/helm/mx-app/.helmignore b/helm/mx-app/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/helm/mx-app/Chart.yaml b/helm/mx-app/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mx-app
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
diff --git a/helm/mx-app/grafana-dashboards-configmap.yaml b/helm/mx-app/grafana-dashboards-configmap.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboards
+  labels:
+    app: grafana
+data:
+  prometheus.yaml: |
+{{ .Files.Get "../../../monitoring/grafana/dashboards/prometheus.yml" | indent 2 }}
+  template_service_dashboard.json: |
+{{ .Files.Get "../../../monitoring/grafana/dashboards/template_service_dashboard.json" | indent 2 }}
diff --git a/helm/mx-app/grafana-datasource-configmap.yaml b/helm/mx-app/grafana-datasource-configmap.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasource
+  labels:
+    app: grafana
+data:
+  prometheus.yaml: |
+{{ .Files.Get "../../../monitoring/grafana/datasources/prometheus.yml" | indent 2 }}
diff --git a/helm/mx-app/grafana-deployment.yaml b/helm/mx-app/grafana-deployment.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      containers:
+        - name: grafana
+          image: grafana/grafana
+          env:
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              value: {{ .Values.grafana.adminPassword }}
+            - name: GF_SECURITY_ADMIN_USER
+              value: {{ .Values.grafana.adminUser }}
+          volumeMounts:
+            - name: datasource-volume
+              mountPath: /etc/grafana/provisioning/datasources
+            - name: dashboard-volume
+              mountPath: /etc/grafana/provisioning/dashboards
+      volumes:
+        - name: datasource-volume
+          configMap:
+            name: grafana-datasource
+        - name: dashboard-volume
+          configMap:
+            name: grafana-dashboards