Merge pull request #12 from jembi/CU-86c19daeq_Add-validations-for-op…

…enhim-config

CU-86c19daeq_Add-validations-for-openhim-config

README.md

@@ -77,3 +77,27 @@ MINIO_SECRETE_KEY:
 ``` bash
 MINIO_BUCKET=climate-mediator,
 ```
+
+## Creating Buckets
+
+### Validation Rules for Creating a Bucket
+
+When creating a bucket the name must follow these following rules:
+> Bucket names must be between 3 (min) and 63 (max) characters long.
+> Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-).
+> Bucket names must not start with the prefix xn--.
+> Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names.
+
+### Enabling Automatic Bucket Creation Through API
+
+To allow automatic creation of the bucket if it does not exist, include the createBucketIfNotExists query parameter and set it to true. This will ensure the bucket is created with the specified name if it is not already present.
+
+```bash
+/upload?bucket=:name&createBucketIfNotExists=true
+```
+
+This optional parameter simplifies the process by eliminating the need to manually create buckets beforehand.
+
+### Enabling Automatic Bucket Creation Through OpenHIM Console
+
+Navigate to `/mediators/urn:mediator:climate-mediator` and click the gear icon, next click the green button that states `Minio Buckets Registry` and add the bucket name and region to the two form elements that appear. Finally click `Save Changes`, the newly entered buckets should appear withing minio momentary.

src/index.ts

@@ -1,10 +1,9 @@
 import express from 'express';
-import path from 'path';
 import { getConfig } from './config/config';
 import logger from './logger';
 import routes from './routes/index';
-import { getRegisteredBuckets, setupMediator } from './openhim/openhim';
-import { createMinioBucketListeners, ensureBucketExists } from './utils/minioClient';
+import { getMediatorConfig, initializeBuckets, setupMediator } from './openhim/openhim';
+import { MinioBucketsRegistry } from './types/mediatorConfig';
 
 const app = express();
 
@@ -15,15 +14,16 @@ app.listen(getConfig().port, async () => {
 
   if (getConfig().runningMode !== 'testing' && getConfig().registerMediator) {
     await setupMediator();
-  }
-
-  const buckets = await getRegisteredBuckets();
 
-  buckets.length === 0 && logger.warn('No buckets specified in the configuration');
-
-  for await (const { bucket, region } of buckets) {
-    await ensureBucketExists(bucket, region, true);
+    const mediatorConfig = await getMediatorConfig();
+    if (mediatorConfig) {
+      await initializeBuckets(
+        mediatorConfig.config?.minio_buckets_registry as MinioBucketsRegistry[]
+      );
+    } else {
+      logger.warn('Failed to fetch mediator config, skipping bucket initialization');
+    }
+  } else {
+    logger.info('Running in testing mode, skipping mediator setup');
   }
-
-  createMinioBucketListeners(buckets.map((bucket) => bucket.bucket));
 });

src/openhim/mediatorConfig.json

@@ -40,7 +40,7 @@
     {
       "param": "minio_buckets_registry",
       "displayName": "Minio Buckets Registry",
-      "description": "The available Minio buckets and their configurations",
+      "description": "The available Minio buckets and their configurations (Note: The names provided must be between 3 and 63 characters long, and can only contain lowercase letters, numbers, dots (.), and hyphens (-))",
       "type": "struct",
       "array": true,
       "template": [

src/openhim/openhim.ts

@@ -1,12 +1,13 @@
 import logger from '../logger';
 import { MediatorConfig, MinioBucketsRegistry } from '../types/mediatorConfig';
 import { RequestOptions } from '../types/request';
-import { getConfig } from '../config/config';
+import { Config, getConfig } from '../config/config';
 import axios, { AxiosError } from 'axios';
 import https from 'https';
 import { activateHeartbeat, fetchConfig, registerMediator } from 'openhim-mediator-utils';
 import { Bucket, createMinioBucketListeners, ensureBucketExists } from '../utils/minioClient';
 import path from 'path';
+import { validateBucketName } from '../utils/file-validators';
 
 const { openhimUsername, openhimPassword, openhimMediatorUrl, trustSelfSigned, runningMode } =
   getConfig();
@@ -63,15 +64,8 @@ export const setupMediator = async () => {
         });
 
         emitter.on('config', async (config: any) => {
-          logger.info('Received config from OpenHIM');
-
-          const buckets = config.minio_buckets_registry as Bucket[];
-
-          for await (const { bucket, region } of buckets) {
-            await ensureBucketExists(bucket, region, true);
-          }
-
-          createMinioBucketListeners(buckets.map((bucket) => bucket.bucket));
+          logger.debug('Received new configs from OpenHIM');
+          await initializeBuckets(config.minio_buckets_registry);
         });
       });
     });
@@ -80,7 +74,42 @@ export const setupMediator = async () => {
   }
 };
 
-async function getMediatorConfig(): Promise<MediatorConfig | null> {
+/**
+ * Initializes the buckets based on the values in the mediator config
+ * if the bucket is invalid, it will be removed from the config
+ * otherwise, the bucket will be created if it doesn't exist
+ * and the listeners will be created for the valid buckets
+ *
+ * @param mediatorConfig - The mediator config
+ */
+export async function initializeBuckets(buckets: MinioBucketsRegistry[]) {
+  if (!buckets) {
+    logger.error('No buckets found in mediator config');
+    return;
+  }
+
+  const validBuckets: string[] = [];
+  const invalidBuckets: string[] = [];
+
+  for await (const { bucket, region } of buckets) {
+    if (!validateBucketName(bucket)) {
+      logger.error(`Invalid bucket name ${bucket}, skipping`);
+      invalidBuckets.push(bucket);
+    } else {
+      await ensureBucketExists(bucket, region, true);
+      validBuckets.push(bucket);
+    }
+  }
+
+  await createMinioBucketListeners(validBuckets);
+
+  if (invalidBuckets.length > 0) {
+    await removeBucket(invalidBuckets);
+    logger.info(`Removed ${invalidBuckets.length} invalid buckets`);
+  }
+}
+
+export async function getMediatorConfig(): Promise<MediatorConfig | null> {
   logger.debug('Fetching mediator config from OpenHIM');
   const mediatorConfig = resolveMediatorConfig();
   const openhimConfig = resolveOpenhimConfig(mediatorConfig.urn);
@@ -152,27 +181,6 @@ async function putMediatorConfig(mediatorUrn: string, mediatorConfig: MinioBucke
   }
 }
 
-export async function getRegisteredBuckets(): Promise<Bucket[]> {
-  if (runningMode === 'testing') {
-    logger.info('Running in testing mode, reading buckets from ENV');
-    const buckets = getConfig().minio.buckets.split(',');
-    return buckets.map((bucket) => ({ bucket, region: '' }));
-  }
-
-  logger.info('Fetching registered buckets from OpenHIM');
-  const mediatorConfig = await getMediatorConfig();
-
-  if (!mediatorConfig) {
-    return [];
-  }
-
-  const buckets = mediatorConfig.config?.minio_buckets_registry as Bucket[];
-  if (buckets) {
-    return buckets;
-  }
-  return [];
-}
-
 export async function registerBucket(bucket: string, region?: string) {
   // If we are in testing mode, we don't need to have the registered buckets persisted
   if (runningMode === 'testing') {
@@ -219,3 +227,27 @@ export async function registerBucket(bucket: string, region?: string) {
 
   return true;
 }
+
+export async function removeBucket(buckets: string[]) {
+  const mediatorConfig = await getMediatorConfig();
+
+  if (!mediatorConfig) {
+    logger.error('Mediator config not found in OpenHIM, unable to remove bucket');
+    return false;
+  }
+
+  const existingConfig = mediatorConfig.config;
+
+  if (existingConfig === undefined) {
+    logger.error('Mediator config does not have a config section, unable to remove bucket');
+    return false;
+  }
+
+  const updatedConfig = existingConfig.minio_buckets_registry.filter(
+    (b) => !buckets.includes(b.bucket)
+  );
+
+  await putMediatorConfig(mediatorConfig.urn, updatedConfig);
+
+  return true;
+}

src/routes/index.ts

@@ -1,7 +1,7 @@
 import express from 'express';
 import multer from 'multer';
 import { getConfig } from '../config/config';
-import { getCsvHeaders } from '../utils/file-validators';
+import { getCsvHeaders, validateBucketName } from '../utils/file-validators';
 import logger from '../logger';
 import fs from 'fs/promises';
 import path from 'path';
@@ -101,15 +101,6 @@ const handleJsonFile = (file: Express.Multer.File): UploadResponse => {
   return createSuccessResponse('JSON_VALID', 'JSON file is valid - Future implementation');
 };
 
-const validateBucketName = (bucket: string): boolean => {
-  // Bucket names must be between 3 (min) and 63 (max) characters long.
-  // Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-).
-  // Bucket names must not start with the prefix xn--.
-  // Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names.
-  const regex = new RegExp(/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/);
-  return regex.test(bucket);
-};
-
 // Main route handler
 routes.post('/upload', upload.single('file'), async (req, res) => {
   try {
@@ -147,7 +138,9 @@ routes.post('/upload', upload.single('file'), async (req, res) => {
         ? await handleCsvFile(file, bucket, region)
         : handleJsonFile(file);
 
-    createBucketIfNotExists && (await registerBucket(bucket, region));
+    if (createBucketIfNotExists && getConfig().runningMode !== 'testing') {
+      await registerBucket(bucket, region);
+    }
 
     const statusCode = response.status === 'success' ? 201 : 400;
     return res.status(statusCode).json(response);

src/utils/file-validators.ts

@@ -22,3 +22,12 @@ export function getCsvHeaders(file: Buffer) {
 
   return columns;
 }
+
+export function validateBucketName(bucket: string): boolean {
+  // Bucket names must be between 3 (min) and 63 (max) characters long.
+  // Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-).
+  // Bucket names must not start with the prefix xn--.
+  // Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names.
+  const regex = new RegExp(/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/);
+  return regex.test(bucket);
+}

src/utils/minioClient.ts

@@ -172,7 +172,7 @@ export async function uploadToMinio(
 export async function createMinioBucketListeners(listOfBuckets: string[]) {
   for (const bucket of listOfBuckets) {
     if (registeredBuckets.has(bucket)) {
-      logger.info(`Bucket ${bucket} already registered`);
+      logger.debug(`Bucket ${bucket} already registered`);
       continue;
     }