Merge pull request #159 from konecty/feat/s3-storage

Feat: S3 storage
konecty · Aug 13, 2024 · acbf3a3 · acbf3a3
2 parents 17c225b + a9d19eb
commit acbf3a3
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 53 deletions.
diff --git a/src/imports/meta/loadMetaObjects.ts b/src/imports/meta/loadMetaObjects.ts
@@ -139,6 +139,9 @@ function dbWatch() {
 				case 'list':
 					MetaObject.DisplayMeta[fullDocument._id as unknown as string] = fullDocument as unknown as (typeof MetaObject.DisplayMeta)[string];
 					break;
+				case 'namespace':
+					Object.assign(MetaObject.Namespace, fullDocument);
+					break;
 			}
 		}
 		rebuildReferences();

diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts
@@ -35,6 +35,7 @@ const HOST = process.env.HOST ?? '0.0.0.0';
 
 const fastify = Fastify({
 	logger,
+	maxParamLength: 250,
 });
 
 fastify.register(initializeInstrumentation(), { ignoreRoutes: ['/liveness', '/readiness'] });

diff --git a/src/server/routes/rest/file/delete.ts b/src/server/routes/rest/file/delete.ts
@@ -20,8 +20,6 @@ const fileDeleteApi: FastifyPluginCallback = (fastify, _, done) => {
 		async function (req, reply) {
 			const { metaDocumentId: document, recordId, fieldName, fileName } = req.params;
 
-			const namespace = MetaObject.Namespace.name;
-
 			const authTokenId = getAuthTokenIdFromReq(req);
 
 			const { success, data: user, errors } = (await getUserSafe(authTokenId)) as any;
@@ -54,7 +52,7 @@ const fileDeleteApi: FastifyPluginCallback = (fastify, _, done) => {
 					await s3.send(
 						new DeleteObjectCommand({
 							Bucket: MetaObject.Namespace.storage.bucket,
-							Key: `${namespace}/${document}/${recordId}/${fieldName}/${fileName}`,
+							Key: `${document}/${recordId}/${fieldName}/${fileName}`,
 						}),
 					);
 				} catch (e) {
@@ -65,7 +63,7 @@ const fileDeleteApi: FastifyPluginCallback = (fastify, _, done) => {
 					await s3.send(
 						new DeleteObjectCommand({
 							Bucket: MetaObject.Namespace.storage.bucket,
-							Key: `${namespace}/${document}/${recordId}/${fieldName}/thumbnail/${fileName}`,
+							Key: `${document}/${recordId}/${fieldName}/thumbnail/${fileName}`,
 						}),
 					);
 				} catch (e) {
@@ -77,33 +75,17 @@ const fileDeleteApi: FastifyPluginCallback = (fastify, _, done) => {
 						await s3.send(
 							new DeleteObjectCommand({
 								Bucket: MetaObject.Namespace.storage.bucket,
-								Key: `${namespace}/${document}/${recordId}/${fieldName}/watermark/${fileName}`,
+								Key: `${document}/${recordId}/${fieldName}/watermark/${fileName}`,
 							}),
 						);
 					} catch (e) {
 						logger.error(e, `Error deleting watermark file ${fileName} from S3`);
 					}
 				}
 			} else {
-				const fullPath = path.join(MetaObject.Namespace.storage?.directory ?? '/tmp', namespace, document, recordId, fieldName, decodeURIComponent(fileName));
-				const thumbnailFullPath = path.join(
-					MetaObject.Namespace.storage?.directory ?? '/tmp',
-					namespace,
-					document,
-					recordId,
-					fieldName,
-					'thumbnail',
-					decodeURIComponent(fileName),
-				);
-				const watermarkFullPath = path.join(
-					MetaObject.Namespace.storage?.directory ?? '/tmp',
-					namespace,
-					document,
-					recordId,
-					fieldName,
-					'watermark',
-					decodeURIComponent(fileName),
-				);
+				const fullPath = path.join(MetaObject.Namespace.storage?.directory ?? '/tmp', document, recordId, fieldName, decodeURIComponent(fileName));
+				const thumbnailFullPath = path.join(MetaObject.Namespace.storage?.directory ?? '/tmp', document, recordId, fieldName, 'thumbnail', decodeURIComponent(fileName));
+				const watermarkFullPath = path.join(MetaObject.Namespace.storage?.directory ?? '/tmp', document, recordId, fieldName, 'watermark', decodeURIComponent(fileName));
 				try {
 					unlink(fullPath);
 				} catch (error) {

diff --git a/src/server/routes/rest/file/download.ts b/src/server/routes/rest/file/download.ts
@@ -1,10 +1,9 @@
 import { FastifyPluginCallback, FastifyReply, FastifyRequest } from 'fastify';
 import fp from 'fastify-plugin';
 
-import { pathToRegexp } from 'path-to-regexp';
 import path from 'path';
+import { pathToRegexp } from 'path-to-regexp';
 
-import { MetaObject } from '@imports/model/MetaObject';
 import { logger } from '@imports/utils/logger';
 
 import { sendFile } from './sendFile';
@@ -31,21 +30,20 @@ async function fileDownloadFn(
 	reply: FastifyReply,
 ) {
 	const incomingPath = req.params['*'];
-	const namespace = MetaObject.Namespace.name;
 
 	if (downloadUrlRegex.test(incomingPath)) {
 		logger.trace(`DOWNLOAD_URL_PATTERN ${incomingPath}`);
 		const [, document, code, fieldName, fileName] = downloadUrlRegex.exec(incomingPath) ?? [];
 
-		const destination = path.join(namespace, document, code, fieldName, fileName);
+		const destination = path.join(document, code, fieldName, fileName);
 		return sendFile(destination, reply);
 	}
 
 	if (legacyDownloadUrlRegex.test(incomingPath)) {
 		logger.trace(`LEGACY_DOWNLOAD_URL_PATTERN ${incomingPath}`);
-		const [, , document, code, fieldName, fileName] = legacyDownloadUrlRegex.exec(incomingPath) ?? [];
+		const [, , , document, code, fieldName, fileName] = legacyDownloadUrlRegex.exec(incomingPath) ?? [];
 
-		const destination = path.join(namespace, document, code, fieldName, fileName);
+		const destination = path.join(document, code, fieldName, fileName);
 		return sendFile(destination, reply);
 	}
 

diff --git a/src/server/routes/rest/file/image.ts b/src/server/routes/rest/file/image.ts
@@ -34,13 +34,12 @@ async function imageApiFn(
 	reply: FastifyReply,
 ) {
 	const incomingPath = req.params['*'];
-	const namespace = MetaObject.Namespace.name;
 
 	if (getFullRegex.test(incomingPath)) {
 		logger.trace(`GET_FULL_PATTERN ${incomingPath}`);
 		const [, document, recordId, fieldName, fileName] = getFullRegex.exec(incomingPath) ?? [];
 
-		const destination = path.join(namespace, document, recordId, fieldName, fileName);
+		const destination = path.join(document, recordId, fieldName, fileName);
 		return sendFile(destination, reply);
 	}
 
@@ -55,7 +54,7 @@ async function imageApiFn(
 				wm: 'watermark',
 			};
 
-			const destination = path.join(...([namespace, document, recordId, fieldName, dirEnum[style], fileName].filter(Boolean) as string[]));
+			const destination = path.join(...([document, recordId, fieldName, dirEnum[style], fileName].filter(Boolean) as string[]));
 			return sendFile(destination, reply);
 		}
 	}
@@ -70,12 +69,13 @@ async function imageApiFn(
 		const getImagePath = () => {
 			const maxDimension = Math.max(parseInt(width, 10), parseInt(height, 10));
 			if (maxDimension <= thumbnailSize) {
-				return path.join(namespace, document, recordId, fieldName, 'thumbnail', fileName);
+				const filePath = path.join(document, recordId, fieldName, 'thumbnail', fileName);
+				return /\.jpe?g$/.test(fileName) ? filePath : `${filePath}.jpeg`;
 			}
 			if (preprocess != null) {
-				return path.join(namespace, document, recordId, fieldName, 'watermark', fileName);
+				return path.join(document, recordId, fieldName, 'watermark', fileName);
 			}
-			return path.join(namespace, document, recordId, fieldName, fileName);
+			return path.join(document, recordId, fieldName, fileName);
 		};
 
 		const destination = getImagePath();
@@ -86,8 +86,7 @@ async function imageApiFn(
 		logger.trace(`LEGACY_FULL_FILE_URL_PATTERN ${incomingPath}`);
 		const [, , preprocess, document, recordId, fieldName, fileName] = legacyFullFileUrlRegex.exec(incomingPath) ?? [];
 
-		const destination =
-			preprocess != null ? path.join(namespace, document, recordId, fieldName, 'watermark', fileName) : path.join(namespace, document, recordId, fieldName, fileName);
+		const destination = preprocess != null ? path.join(document, recordId, fieldName, 'watermark', fileName) : path.join(document, recordId, fieldName, fileName);
 
 		return sendFile(destination, reply);
 	}

diff --git a/src/server/routes/rest/file/sendFile.ts b/src/server/routes/rest/file/sendFile.ts
@@ -1,6 +1,6 @@
 import { FastifyReply } from 'fastify';
 
-import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { GetObjectCommand, NoSuchKey, S3Client } from '@aws-sdk/client-s3';
 
 import crypto from 'crypto';
 import mime from 'mime-types';
@@ -18,8 +18,8 @@ import { ALLOWED_CORS_FILE_TYPES, DEFAULT_EXPIRATION } from '@imports/consts';
 export async function sendFile(filePath: string, reply: FastifyReply) {
 	const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
 	if (MetaObject.Namespace.storage?.type === 's3') {
-		logger.trace(`Proxying file ${MetaObject.Namespace.storage.publicUrl}/${filePath} from S3`);
 		if (MetaObject.Namespace.storage?.publicUrl != null) {
+			logger.trace(`Proxying file ${MetaObject.Namespace.storage.publicUrl}/${filePath} from S3`);
 			const { statusCode, headers, body } = await request(`${MetaObject.Namespace.storage.publicUrl}/${filePath}`);
 
 			if (statusCode !== 200) {
@@ -80,7 +80,7 @@ export async function sendFile(filePath: string, reply: FastifyReply) {
 							Bucket: MetaObject.Namespace.storage.bucket,
 							Key: filePath,
 						},
-						result: s3Result,
+						result: s3Result.$metadata,
 					},
 					`Proxying file ${filePath} from S3`,
 				);
@@ -96,11 +96,10 @@ export async function sendFile(filePath: string, reply: FastifyReply) {
 					})
 					.send(Body);
 			} catch (error) {
-				logger.error(error, `Error proxying file ${filePath} from S3`);
-				if ((error as any).type === 'NoSuchKey') {
+				if (error instanceof NoSuchKey) {
 					return reply.status(404).send('Not found');
 				}
-
+				logger.error(error, `Error proxying file ${filePath} from S3`);
 				return reply.status(500).send('Error retrieving file');
 			}
 		}

diff --git a/src/server/routes/rest/file/upload.ts b/src/server/routes/rest/file/upload.ts
@@ -70,11 +70,11 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 			const data = await req.file();
 
 			if (data == null) {
-				return errorReturn('[${document}] No file sent');
+				return errorReturn(`[${document}] No file sent`);
 			}
 
 			const contentType = data.mimetype;
-			const fileName = sanitizeFilename(decodeURIComponent(data.filename));
+			const fileName = encodeURIComponent(sanitizeFilename(decodeURIComponent(data.filename)));
 
 			let fileContent = await data.toBuffer();
 
@@ -133,7 +133,7 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 					.resize({
 						width: MetaObject.Namespace.storage?.thumbnail?.size ?? DEFAULT_THUMBNAIL_SIZE,
 						height: MetaObject.Namespace.storage?.thumbnail?.size ?? DEFAULT_THUMBNAIL_SIZE,
-						fit: 'inside',
+						fit: 'cover',
 					})
 					.jpeg({
 						quality: MetaObject.Namespace.storage?.jpeg?.quality ?? DEFAULT_JPEG_QUALITY,
@@ -214,7 +214,7 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 					const s3Result = await s3.send(
 						new PutObjectCommand({
 							Bucket: bucket,
-							Key: `${namespace}/${directory}/${name}`,
+							Key: `${directory}/${name}`,
 							ContentType: contentType,
 							Body: content,
 						}),
@@ -226,7 +226,7 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 						{
 							params: {
 								Bucket: bucket,
-								Key: `${namespace}/${directory}/${fileName}`,
+								Key: `${directory}/${fileName}`,
 								ContentType: contentType,
 							},
 							result: s3Result,
@@ -244,8 +244,8 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 				const storageDirectory = MetaObject.Namespace.storage?.directory ?? '/tmp';
 
 				await BluebirdPromise.each(filesToSave, async ({ name, content }) => {
-					await mkdirp(path.dirname(join(storageDirectory, namespace, directory, name)));
-					const filePath = join(storageDirectory, namespace, directory, name);
+					await mkdirp(path.dirname(join(storageDirectory, directory, name)));
+					const filePath = join(storageDirectory, directory, name);
 					await writeFile(filePath, content);
 				});
 			}
@@ -268,15 +268,15 @@ const fileUploadApi: FastifyPluginCallback = (fastify, _, done) => {
 						await s3.send(
 							new DeleteObjectCommand({
 								Bucket: bucket,
-								Key: `${namespace}/${directory}/${name}`,
+								Key: `${directory}/${name}`,
 								VersionId: fileData.version,
 							}),
 						);
 					});
 				} else {
 					const storageDirectory = MetaObject.Namespace.storage?.directory ?? '/tmp';
 					await BluebirdPromise.each(filesToSave, async ({ name }) => {
-						await unlink(join(storageDirectory, namespace, directory, name));
+						await unlink(join(storageDirectory, directory, name));
 					});
 				}
 				return reply.send(coreResponse);
-Original file line number
+Diff line change
@@ Expand Up / @@ -35,6 +35,7 @@ const HOST = process.env.HOST ?? '0.0.0.0'; @@
     const fastify = Fastify({
     	logger,
+    	maxParamLength: 250,
     });
     fastify.register(initializeInstrumentation(), { ignoreRoutes: ['/liveness', '/readiness'] });
@@ Expand Down @@