From e1049c4968a0877a91ed2686652394fe845062cf Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Mon, 9 Dec 2024 15:23:35 +0200 Subject: [PATCH 1/3] updating file name validation --- src/openhim/openhim.ts | 70 ++++++++++++++++++++++++++++++------ src/routes/index.ts | 11 +----- src/utils/file-validators.ts | 9 +++++ 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/src/openhim/openhim.ts b/src/openhim/openhim.ts index 8d2248a..3baad64 100644 --- a/src/openhim/openhim.ts +++ b/src/openhim/openhim.ts @@ -7,6 +7,7 @@ import https from 'https'; import { activateHeartbeat, fetchConfig, registerMediator } from 'openhim-mediator-utils'; import { Bucket, createMinioBucketListeners, ensureBucketExists } from '../utils/minioClient'; import path from 'path'; +import { validateBucketName } from '../utils/file-validators'; const { openhimUsername, openhimPassword, openhimMediatorUrl, trustSelfSigned, runningMode } = getConfig(); @@ -65,13 +66,7 @@ export const setupMediator = async () => { emitter.on('config', async (config: any) => { logger.info('Received config from OpenHIM'); - const buckets = config.minio_buckets_registry as Bucket[]; - - for await (const { bucket, region } of buckets) { - await ensureBucketExists(bucket, region); - } - - createMinioBucketListeners(buckets.map((bucket) => bucket.bucket)); + await initializeBuckets(config); }); }); }); @@ -80,6 +75,37 @@ export const setupMediator = async () => { } }; +/** + * Initializes the buckets based on the values in the mediator config + * if the bucket is invalid, it will be removed from the config + * otherwise, the bucket will be created if it doesn't exist + * and the listeners will be created for the valid buckets + * + * @param mediatorConfig - The mediator config + */ +async function initializeBuckets(mediatorConfig: MediatorConfig) { + const bucketsFromOpenhimConfig = mediatorConfig.config?.minio_buckets_registry as Bucket[]; + const validBuckets: string[] = []; + const invalidBuckets: string[] = []; + + for await (const { bucket, region } of bucketsFromOpenhimConfig) { + if (!validateBucketName(bucket)) { + logger.error(`Invalid bucket name ${bucket}, skipping`); + invalidBuckets.push(bucket); + } else { + await ensureBucketExists(bucket, region); + validBuckets.push(bucket); + } + } + + await createMinioBucketListeners(validBuckets); + + if (invalidBuckets.length > 0) { + await removeBucket(invalidBuckets); + logger.info(`Removed ${invalidBuckets.length} invalid buckets`); + } +} + async function getMediatorConfig(): Promise { logger.debug('Fetching mediator config from OpenHIM'); const mediatorConfig = resolveMediatorConfig(); @@ -166,9 +192,9 @@ export async function getRegisteredBuckets(): Promise { return []; } - const buckets = mediatorConfig.config?.minio_buckets_registry as Bucket[]; - if (buckets) { - return buckets; + if (mediatorConfig) { + await initializeBuckets(mediatorConfig); + return mediatorConfig.config?.minio_buckets_registry as Bucket[]; } return []; } @@ -219,3 +245,27 @@ export async function registerBucket(bucket: string, region?: string) { return true; } + +export async function removeBucket(buckets: string[]) { + const mediatorConfig = await getMediatorConfig(); + + if (!mediatorConfig) { + logger.error('Mediator config not found in OpenHIM, unable to remove bucket'); + return false; + } + + const existingConfig = mediatorConfig.config; + + if (existingConfig === undefined) { + logger.error('Mediator config does not have a config section, unable to remove bucket'); + return false; + } + + const updatedConfig = existingConfig.minio_buckets_registry.filter( + (b) => !buckets.includes(b.bucket) + ); + + await putMediatorConfig(mediatorConfig.urn, updatedConfig); + + return true; +} diff --git a/src/routes/index.ts b/src/routes/index.ts index fed182c..da9db32 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,7 +1,7 @@ import express from 'express'; import multer from 'multer'; import { getConfig } from '../config/config'; -import { getCsvHeaders } from '../utils/file-validators'; +import { getCsvHeaders, validateBucketName } from '../utils/file-validators'; import logger from '../logger'; import fs from 'fs/promises'; import path from 'path'; @@ -101,15 +101,6 @@ const handleJsonFile = (file: Express.Multer.File): UploadResponse => { return createSuccessResponse('JSON_VALID', 'JSON file is valid - Future implementation'); }; -const validateBucketName = (bucket: string): boolean => { - // Bucket names must be between 3 (min) and 63 (max) characters long. - // Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-). - // Bucket names must not start with the prefix xn--. - // Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names. - const regex = new RegExp(/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/); - return regex.test(bucket); -}; - // Main route handler routes.post('/upload', upload.single('file'), async (req, res) => { try { diff --git a/src/utils/file-validators.ts b/src/utils/file-validators.ts index a494578..89d4945 100644 --- a/src/utils/file-validators.ts +++ b/src/utils/file-validators.ts @@ -22,3 +22,12 @@ export function getCsvHeaders(file: Buffer) { return columns; } + +export function validateBucketName(bucket: string): boolean { + // Bucket names must be between 3 (min) and 63 (max) characters long. + // Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-). + // Bucket names must not start with the prefix xn--. + // Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names. + const regex = new RegExp(/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/); + return regex.test(bucket); +} From 6d4709106f79c713afc03e09553617350324c5fc Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Tue, 10 Dec 2024 13:48:12 +0200 Subject: [PATCH 2/3] fx: bug when listening for updates to config --- src/index.ts | 21 +++++++++------------ src/openhim/openhim.ts | 21 +++++++++++++++------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/index.ts b/src/index.ts index 229618f..c23f423 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,8 @@ import express from 'express'; -import path from 'path'; import { getConfig } from './config/config'; import logger from './logger'; import routes from './routes/index'; -import { getRegisteredBuckets, setupMediator } from './openhim/openhim'; -import { createMinioBucketListeners, ensureBucketExists } from './utils/minioClient'; +import { getMediatorConfig, initializeBuckets, setupMediator } from './openhim/openhim'; const app = express(); @@ -15,15 +13,14 @@ app.listen(getConfig().port, async () => { if (getConfig().runningMode !== 'testing' && getConfig().registerMediator) { await setupMediator(); - } - - const buckets = await getRegisteredBuckets(); - buckets.length === 0 && logger.warn('No buckets specified in the configuration'); - - for await (const { bucket, region } of buckets) { - await ensureBucketExists(bucket, region); + const mediatorConfig = await getMediatorConfig(); + if (mediatorConfig) { + await initializeBuckets(mediatorConfig); + } else { + logger.error('Failed to fetch mediator config'); + } + } else { + logger.info('Running in testing mode, skipping mediator setup'); } - - createMinioBucketListeners(buckets.map((bucket) => bucket.bucket)); }); diff --git a/src/openhim/openhim.ts b/src/openhim/openhim.ts index 3baad64..dff024d 100644 --- a/src/openhim/openhim.ts +++ b/src/openhim/openhim.ts @@ -64,9 +64,18 @@ export const setupMediator = async () => { }); emitter.on('config', async (config: any) => { - logger.info('Received config from OpenHIM'); - - await initializeBuckets(config); + const mediatorConfig = { + config: { + minio_buckets_registry: config.minio_buckets_registry, + }, + defaultChannelConfig: [], + endpoints: [], + urn: config.urn, + version: config.version, + name: config.name, + description: config.description, + }; + await initializeBuckets(mediatorConfig); }); }); }); @@ -83,7 +92,7 @@ export const setupMediator = async () => { * * @param mediatorConfig - The mediator config */ -async function initializeBuckets(mediatorConfig: MediatorConfig) { +export async function initializeBuckets(mediatorConfig: MediatorConfig) { const bucketsFromOpenhimConfig = mediatorConfig.config?.minio_buckets_registry as Bucket[]; const validBuckets: string[] = []; const invalidBuckets: string[] = []; @@ -93,7 +102,7 @@ async function initializeBuckets(mediatorConfig: MediatorConfig) { logger.error(`Invalid bucket name ${bucket}, skipping`); invalidBuckets.push(bucket); } else { - await ensureBucketExists(bucket, region); + await ensureBucketExists(bucket, region, true); validBuckets.push(bucket); } } @@ -106,7 +115,7 @@ async function initializeBuckets(mediatorConfig: MediatorConfig) { } } -async function getMediatorConfig(): Promise { +export async function getMediatorConfig(): Promise { logger.debug('Fetching mediator config from OpenHIM'); const mediatorConfig = resolveMediatorConfig(); const openhimConfig = resolveOpenhimConfig(mediatorConfig.urn); From 96fef7e7afa5163cfc862122574540b653a3d543 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Tue, 10 Dec 2024 15:09:13 +0200 Subject: [PATCH 3/3] clean up and updating docs --- README.md | 24 +++++++++++++++++ src/index.ts | 7 +++-- src/openhim/mediatorConfig.json | 2 +- src/openhim/openhim.ts | 46 +++++++-------------------------- src/routes/index.ts | 4 ++- src/utils/minioClient.ts | 2 +- 6 files changed, 43 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index b0d22f3..ba993d5 100644 --- a/README.md +++ b/README.md @@ -77,3 +77,27 @@ MINIO_SECRETE_KEY: ``` bash MINIO_BUCKET=climate-mediator, ``` + +## Creating Buckets + +### Validation Rules for Creating a Bucket + +When creating a bucket the name must follow these following rules: +> Bucket names must be between 3 (min) and 63 (max) characters long. +> Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-). +> Bucket names must not start with the prefix xn--. +> Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names. + +### Enabling Automatic Bucket Creation Through API + +To allow automatic creation of the bucket if it does not exist, include the createBucketIfNotExists query parameter and set it to true. This will ensure the bucket is created with the specified name if it is not already present. + +```bash +/upload?bucket=:name&createBucketIfNotExists=true +``` + +This optional parameter simplifies the process by eliminating the need to manually create buckets beforehand. + +### Enabling Automatic Bucket Creation Through OpenHIM Console + +Navigate to `/mediators/urn:mediator:climate-mediator` and click the gear icon, next click the green button that states `Minio Buckets Registry` and add the bucket name and region to the two form elements that appear. Finally click `Save Changes`, the newly entered buckets should appear withing minio momentary. diff --git a/src/index.ts b/src/index.ts index c23f423..61a8e24 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { getConfig } from './config/config'; import logger from './logger'; import routes from './routes/index'; import { getMediatorConfig, initializeBuckets, setupMediator } from './openhim/openhim'; +import { MinioBucketsRegistry } from './types/mediatorConfig'; const app = express(); @@ -16,9 +17,11 @@ app.listen(getConfig().port, async () => { const mediatorConfig = await getMediatorConfig(); if (mediatorConfig) { - await initializeBuckets(mediatorConfig); + await initializeBuckets( + mediatorConfig.config?.minio_buckets_registry as MinioBucketsRegistry[] + ); } else { - logger.error('Failed to fetch mediator config'); + logger.warn('Failed to fetch mediator config, skipping bucket initialization'); } } else { logger.info('Running in testing mode, skipping mediator setup'); diff --git a/src/openhim/mediatorConfig.json b/src/openhim/mediatorConfig.json index 09f6880..15e0b94 100644 --- a/src/openhim/mediatorConfig.json +++ b/src/openhim/mediatorConfig.json @@ -40,7 +40,7 @@ { "param": "minio_buckets_registry", "displayName": "Minio Buckets Registry", - "description": "The available Minio buckets and their configurations", + "description": "The available Minio buckets and their configurations (Note: The names provided must be between 3 and 63 characters long, and can only contain lowercase letters, numbers, dots (.), and hyphens (-))", "type": "struct", "array": true, "template": [ diff --git a/src/openhim/openhim.ts b/src/openhim/openhim.ts index 5d32ce6..4941194 100644 --- a/src/openhim/openhim.ts +++ b/src/openhim/openhim.ts @@ -1,7 +1,7 @@ import logger from '../logger'; import { MediatorConfig, MinioBucketsRegistry } from '../types/mediatorConfig'; import { RequestOptions } from '../types/request'; -import { getConfig } from '../config/config'; +import { Config, getConfig } from '../config/config'; import axios, { AxiosError } from 'axios'; import https from 'https'; import { activateHeartbeat, fetchConfig, registerMediator } from 'openhim-mediator-utils'; @@ -65,18 +65,7 @@ export const setupMediator = async () => { emitter.on('config', async (config: any) => { logger.debug('Received new configs from OpenHIM'); - const mediatorConfig = { - config: { - minio_buckets_registry: config.minio_buckets_registry, - }, - defaultChannelConfig: [], - endpoints: [], - urn: config.urn, - version: config.version, - name: config.name, - description: config.description, - }; - await initializeBuckets(mediatorConfig); + await initializeBuckets(config.minio_buckets_registry); }); }); }); @@ -93,12 +82,16 @@ export const setupMediator = async () => { * * @param mediatorConfig - The mediator config */ -export async function initializeBuckets(mediatorConfig: MediatorConfig) { - const bucketsFromOpenhimConfig = mediatorConfig.config?.minio_buckets_registry as Bucket[]; +export async function initializeBuckets(buckets: MinioBucketsRegistry[]) { + if (!buckets) { + logger.error('No buckets found in mediator config'); + return; + } + const validBuckets: string[] = []; const invalidBuckets: string[] = []; - for await (const { bucket, region } of bucketsFromOpenhimConfig) { + for await (const { bucket, region } of buckets) { if (!validateBucketName(bucket)) { logger.error(`Invalid bucket name ${bucket}, skipping`); invalidBuckets.push(bucket); @@ -188,27 +181,6 @@ async function putMediatorConfig(mediatorUrn: string, mediatorConfig: MinioBucke } } -export async function getRegisteredBuckets(): Promise { - if (runningMode === 'testing') { - logger.info('Running in testing mode, reading buckets from ENV'); - const buckets = getConfig().minio.buckets.split(','); - return buckets.map((bucket) => ({ bucket, region: '' })); - } - - logger.info('Fetching registered buckets from OpenHIM'); - const mediatorConfig = await getMediatorConfig(); - - if (!mediatorConfig) { - return []; - } - - if (mediatorConfig) { - await initializeBuckets(mediatorConfig); - return mediatorConfig.config?.minio_buckets_registry as Bucket[]; - } - return []; -} - export async function registerBucket(bucket: string, region?: string) { // If we are in testing mode, we don't need to have the registered buckets persisted if (runningMode === 'testing') { diff --git a/src/routes/index.ts b/src/routes/index.ts index da9db32..92f7d89 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -138,7 +138,9 @@ routes.post('/upload', upload.single('file'), async (req, res) => { ? await handleCsvFile(file, bucket, region) : handleJsonFile(file); - createBucketIfNotExists && (await registerBucket(bucket, region)); + if (createBucketIfNotExists && getConfig().runningMode !== 'testing') { + await registerBucket(bucket, region); + } const statusCode = response.status === 'success' ? 201 : 400; return res.status(statusCode).json(response); diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts index e2663db..f1d1ebc 100644 --- a/src/utils/minioClient.ts +++ b/src/utils/minioClient.ts @@ -172,7 +172,7 @@ export async function uploadToMinio( export async function createMinioBucketListeners(listOfBuckets: string[]) { for (const bucket of listOfBuckets) { if (registeredBuckets.has(bucket)) { - logger.info(`Bucket ${bucket} already registered`); + logger.debug(`Bucket ${bucket} already registered`); continue; }