diff --git a/package-lock.json b/package-lock.json index a4cb20c..47cfa78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,14 +12,14 @@ "@typescript-eslint/eslint-plugin": "5.0.0", "@typescript-eslint/parser": "5.0.0", "eslint": "8.0.1", - "eslint-config-prettier": "8.3.0", - "eslint-plugin-prettier": "4.0.0", + "eslint-config-prettier": "9.1.0", + "eslint-plugin-prettier": "5.2.1", "husky": "8.0.1", "jest": "29.1.2", "lerna": "4.0.0", - "prettier": "2.4.1", + "prettier": "3.3.3", "ts-jest": "29.0.3", - "typescript": "4.8.4" + "typescript": "5.5.4" } }, "node_modules/@ampproject/remapping": { @@ -2480,6 +2480,19 @@ "@octokit/openapi-types": "^11.2.0" } }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/@sinclair/typebox": { "version": "0.24.46", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.46.tgz", @@ -4378,10 +4391,11 @@ } }, "node_modules/eslint-config-prettier": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz", - "integrity": "sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, + "license": "MIT", "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -4390,21 +4404,31 @@ } }, "node_modules/eslint-plugin-prettier": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.0.0.tgz", - "integrity": "sha512-98MqmCJ7vJodoQK359bqQWaxOE0CS8paAz/GgjaZLyex4TTk3g9HugoO89EqWCrFiOqn9EVvcoo7gZzONCWVwQ==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.2.1.tgz", + "integrity": "sha512-gH3iR3g4JfF+yYPaJYkN7jEl9QbweL/YfkoRlNnuIEHEz1vHVlCmWOS+eGGiRuzHQXdJFCOTxRgvju9b8VUmrw==", "dev": true, + "license": "MIT", "dependencies": { - "prettier-linter-helpers": "^1.0.0" + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.9.1" }, "engines": { - "node": ">=6.0.0" + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" }, "peerDependencies": { - "eslint": ">=7.28.0", - "prettier": ">=2.0.0" + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": "*", + "prettier": ">=3.0.0" }, "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, "eslint-config-prettier": { "optional": true } @@ -8547,15 +8571,19 @@ } }, "node_modules/prettier": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", - "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", "dev": true, + "license": "MIT", "bin": { - "prettier": "bin-prettier.js" + "prettier": "bin/prettier.cjs" }, "engines": { - "node": ">=10.13.0" + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" } }, "node_modules/prettier-linter-helpers": { @@ -9795,6 +9823,30 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/synckit": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.1.tgz", + "integrity": "sha512-7gr8p9TQP6RAHusBOSLs46F4564ZrjV8xFmw5zCmgmhGUcw2hxsShhJ6CEiHQMgPDwAQ1fWHPM0ypc4RMAig4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/synckit/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "dev": true, + "license": "0BSD" + }, "node_modules/tar": { "version": "6.1.11", "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", @@ -10110,16 +10162,17 @@ } }, "node_modules/typescript": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", - "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=4.2.0" + "node": ">=14.17" } }, "node_modules/uglify-js": { diff --git a/package.json b/package.json index 7c16d13..e166e44 100644 --- a/package.json +++ b/package.json @@ -16,13 +16,13 @@ "@typescript-eslint/eslint-plugin": "5.0.0", "@typescript-eslint/parser": "5.0.0", "eslint": "8.0.1", - "eslint-config-prettier": "8.3.0", - "eslint-plugin-prettier": "4.0.0", + "eslint-config-prettier": "9.1.0", + "eslint-plugin-prettier": "5.2.1", "husky": "8.0.1", "jest": "29.1.2", "lerna": "4.0.0", - "prettier": "2.4.1", + "prettier": "3.3.3", "ts-jest": "29.0.3", - "typescript": "4.8.4" + "typescript": "5.5.4" } -} +} \ No newline at end of file diff --git a/packages/oc-azure-storage-adapter/package-lock.json b/packages/oc-azure-storage-adapter/package-lock.json index a172108..822bc74 100644 --- a/packages/oc-azure-storage-adapter/package-lock.json +++ b/packages/oc-azure-storage-adapter/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "oc-azure-storage-adapter", - "version": "1.1.2", + "version": "1.2.1", "license": "MIT", "dependencies": { "@azure/identity": "^4.2.1", @@ -16,7 +16,7 @@ "lodash": "^4.17.4", "nice-cache": "^0.0.5", "node-dir": "^0.1.17", - "oc-storage-adapters-utils": "2.0.3" + "oc-storage-adapters-utils": "2.0.4" }, "devDependencies": { "@types/fs-extra": "9.0.13", @@ -1097,9 +1097,9 @@ } }, "node_modules/oc-storage-adapters-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/oc-storage-adapters-utils/-/oc-storage-adapters-utils-2.0.3.tgz", - "integrity": "sha512-Zk6TMs+S9AdMhScj3fo5GFpJ0+yHIzabET/y/zbaYHfKs/Fe49T2JDQitfOYruv5in78eoSxinjyc6BaGPJ4kQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/oc-storage-adapters-utils/-/oc-storage-adapters-utils-2.0.4.tgz", + "integrity": "sha512-Lv1kQou/pNnbjmGrwvKJ7qvqzUjL6jqPhD7tnqXBY6wH6B3gvRpR4Sk3NT+lIEXRrip0nZc9wDA0ONFjm4ZYOQ==", "license": "MIT" }, "node_modules/open": { diff --git a/packages/oc-azure-storage-adapter/src/index.ts b/packages/oc-azure-storage-adapter/src/index.ts index 7948775..87c6ed2 100644 --- a/packages/oc-azure-storage-adapter/src/index.ts +++ b/packages/oc-azure-storage-adapter/src/index.ts @@ -1,18 +1,19 @@ import { BlobServiceClient, + type ContainerClient, StorageSharedKeyCredential, - BlockBlobUploadOptions + type BlockBlobUploadOptions } from '@azure/storage-blob'; import Cache from 'nice-cache'; import fs from 'fs-extra'; import { DefaultAzureCredential } from '@azure/identity'; -import nodeDir, { PathsResult } from 'node-dir'; +import nodeDir, { type PathsResult } from 'node-dir'; import { promisify } from 'util'; import { getFileInfo, strings, - StorageAdapter, - StorageAdapterBaseConfig + type StorageAdapter, + type StorageAdapterBaseConfig } from 'oc-storage-adapters-utils'; import path from 'path'; @@ -58,27 +59,29 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { refreshInterval: conf.refreshInterval }); - let client: BlobServiceClient | undefined = undefined; + let privateClient: ContainerClient | undefined = undefined; + let publicClient: ContainerClient | undefined = undefined; const getClient = () => { - if (!client) { - client = new BlobServiceClient( + if (!privateClient || !publicClient) { + const client = new BlobServiceClient( `https://${conf.accountName}.blob.core.windows.net`, conf.accountName && conf.accountKey ? new StorageSharedKeyCredential(conf.accountName, conf.accountKey) : new DefaultAzureCredential() ); + publicClient = client.getContainerClient(conf.publicContainerName); + privateClient = client.getContainerClient(conf.privateContainerName); + + return { publicClient, privateClient }; } - return client; + return { publicClient, privateClient }; }; const getFile = async (filePath: string, force = false) => { const getFromAzure = async () => { - const client = getClient(); - const containerClient = client.getContainerClient( - conf.privateContainerName - ); - const blobClient = containerClient.getBlobClient(filePath); + const { privateClient } = getClient(); + const blobClient = privateClient.getBlobClient(filePath); try { const downloadBlockBlobResponse = await blobClient.download(); const fileContent = ( @@ -136,12 +139,10 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { ? dir : `${dir}/`; - const containerClient = getClient().getContainerClient( - conf.privateContainerName - ); + const { privateClient } = getClient(); const subDirectories = []; - for await (const item of containerClient.listBlobsByHierarchy('/', { + for await (const item of privateClient.listBlobsByHierarchy('/', { prefix: normalisedPath })) { if (item.kind === 'prefix') { @@ -160,7 +161,6 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { const paths = await getPaths(dirInput); const packageJsonFile = path.join(dirInput, 'package.json'); const files = paths.files.filter(file => file !== packageJsonFile); - const client = getClient(); const filesResults = await Promise.all( files.map((file: string) => { @@ -173,8 +173,7 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { return putFile( file, url, - privateFilePatterns.some(r => r.test(relativeFile)), - client + privateFilePatterns.some(r => r.test(relativeFile)) ); }) ); @@ -183,8 +182,7 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { const packageJsonFileResult = await putFile( packageJsonFile, `${dirOutput}/package.json`.replace(/\\/g, '/'), - false, - client + false ); return [...filesResults, packageJsonFileResult]; @@ -193,15 +191,14 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { const putFileContent = async ( fileContent: string | fs.ReadStream, fileName: string, - isPrivate: boolean, - client: BlobServiceClient + isPrivate: boolean ) => { const content = typeof fileContent === 'string' ? Buffer.from(fileContent) : await streamToBuffer(fileContent); - const uploadToAzureContainer = (containerName: string) => { + const uploadToAzureContainer = (client: ContainerClient) => { const fileInfo = getFileInfo(fileName); const blobHTTPHeaders: BlockBlobUploadOptions['blobHTTPHeaders'] = { blobCacheControl: 'public, max-age=31556926' @@ -214,30 +211,57 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { if (fileInfo.gzip) { blobHTTPHeaders.blobContentEncoding = 'gzip'; } - const localClient = client ? client : getClient(); - const containerClient = localClient.getContainerClient(containerName); - const blockBlobClient = containerClient.getBlockBlobClient(fileName); + const blockBlobClient = client.getBlockBlobClient(fileName); return blockBlobClient.uploadData(content, { blobHTTPHeaders }); }; - let result = await uploadToAzureContainer(conf.privateContainerName); + const { publicClient, privateClient } = getClient(); + let result = await uploadToAzureContainer(privateClient); if (!isPrivate) { - result = await uploadToAzureContainer(conf.publicContainerName); + result = await uploadToAzureContainer(publicClient); } return result; }; - const putFile = ( - filePath: string, - fileName: string, - isPrivate: boolean, - client: BlobServiceClient - ) => { + const putFile = (filePath: string, fileName: string, isPrivate: boolean) => { const stream = fs.createReadStream(filePath); - return putFileContent(stream, fileName, isPrivate, client); + return putFileContent(stream, fileName, isPrivate); + }; + + const removeDir = async (dir: string) => { + const removeFromContainer = async (isPrivate: boolean) => { + const { publicClient, privateClient } = getClient(); + const client = isPrivate ? privateClient : publicClient; + const files: string[] = []; + const normalisedPath = + dir.lastIndexOf('/') === dir.length - 1 && dir.length > 0 + ? dir + : `${dir}/`; + + for await (const blob of client.listBlobsFlat({ + prefix: normalisedPath + })) { + files.push(blob.name); + } + + return Promise.all(files.map(file => removeFile(file, isPrivate))); + }; + + return Promise.all([removeFromContainer(true), removeFromContainer(false)]); + }; + + const removeFile = async (filePath: string, isPrivate: boolean) => { + const { publicClient, privateClient } = getClient(); + if (!isPrivate) { + const blockBlobClient = publicClient.getBlockBlobClient(filePath); + await blockBlobClient.delete(); + } + + const blockBlobClient = privateClient.getBlockBlobClient(filePath); + return blockBlobClient.delete(); }; return { @@ -249,6 +273,8 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter { putDir, putFile, putFileContent, + removeFile, + removeDir, adapterType: 'azure-blob-storage', isValid }; diff --git a/packages/oc-gs-storage-adapter/src/index.ts b/packages/oc-gs-storage-adapter/src/index.ts index 83c615a..b382b71 100644 --- a/packages/oc-gs-storage-adapter/src/index.ts +++ b/packages/oc-gs-storage-adapter/src/index.ts @@ -1,13 +1,13 @@ import Cache from 'nice-cache'; import fs from 'fs-extra'; -import nodeDir, { PathsResult } from 'node-dir'; -import { Storage, UploadOptions } from '@google-cloud/storage'; +import nodeDir, { type PathsResult } from 'node-dir'; +import { Storage, type UploadOptions } from '@google-cloud/storage'; import tmp from 'tmp'; import { getFileInfo, strings, - StorageAdapter, - StorageAdapterBaseConfig + type StorageAdapter, + type StorageAdapterBaseConfig } from 'oc-storage-adapters-utils'; import { promisify } from 'util'; import path from 'path'; @@ -241,6 +241,29 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter { } }; + const removeDir = async (dir: string) => { + const removeFromContainer = async () => { + const normalisedPath = + dir.lastIndexOf('/') === dir.length - 1 && dir.length > 0 + ? dir + : `${dir}/`; + + const [files] = await getClient() + .bucket(bucketName) + .getFiles({ prefix: normalisedPath }); + + return Promise.all(files.map(file => file.delete())); + }; + + return removeFromContainer(); + }; + + const removeFile = async (filePath: string) => { + const client = getClient(); + + return client.bucket(bucketName).file(filePath).delete(); + }; + return { getFile, getJson, @@ -250,6 +273,8 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter { putDir, putFile, putFileContent, + removeDir, + removeFile, adapterType: 'gs', isValid }; diff --git a/packages/oc-s3-storage-adapter/src/index.ts b/packages/oc-s3-storage-adapter/src/index.ts index 2652407..5f31cb1 100644 --- a/packages/oc-s3-storage-adapter/src/index.ts +++ b/packages/oc-s3-storage-adapter/src/index.ts @@ -1,11 +1,11 @@ -import { S3, S3ClientConfig } from '@aws-sdk/client-s3'; +import { S3, type S3ClientConfig } from '@aws-sdk/client-s3'; import { NodeHttpHandler, - NodeHttpHandlerOptions + type NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import Cache from 'nice-cache'; import fs from 'fs-extra'; -import nodeDir, { PathsResult } from 'node-dir'; +import nodeDir, { type PathsResult } from 'node-dir'; import _ from 'lodash'; import { promisify } from 'util'; @@ -13,8 +13,8 @@ import { getFileInfo, getNextYear, strings, - StorageAdapter, - StorageAdapterBaseConfig + type StorageAdapter, + type StorageAdapterBaseConfig } from 'oc-storage-adapters-utils'; import path from 'path'; @@ -27,8 +27,8 @@ const getPaths: (path: string) => Promise = promisify( type RequireAllOrNone = ( | Required> // Require all of the given keys. - | Partial> -) & // Require none of the given keys. + | Partial> // Require none of the given keys. +) & Omit; // The rest of the keys. export type S3Config = StorageAdapterBaseConfig & @@ -106,7 +106,7 @@ export default function s3Adapter(conf: S3Config): StorageAdapter { endpoint: conf.endpoint, region, forcePathStyle: s3ForcePathStyle - } + }; if (accessKeyId && secretAccessKey) { configOpts.credentials = { accessKeyId, @@ -181,7 +181,7 @@ export default function s3Adapter(conf: S3Config): StorageAdapter { Delimiter: '/' }); - if (data.CommonPrefixes!.length === 0) { + if (data.CommonPrefixes?.length === 0) { throw { code: strings.errors.STORAGE.DIR_NOT_FOUND_CODE, msg: strings.errors.STORAGE.DIR_NOT_FOUND(dir) @@ -253,12 +253,48 @@ export default function s3Adapter(conf: S3Config): StorageAdapter { }); }; - const putFile = (filePath: string, fileName: string, isPrivate: boolean, client: S3) => { + const putFile = ( + filePath: string, + fileName: string, + isPrivate: boolean, + client: S3 + ) => { const stream = fs.createReadStream(filePath); return putFileContent(stream, fileName, isPrivate, client); }; + const removeDir = async (dir: string) => { + const removeFromContainer = async () => { + const normalisedPath = + dir.lastIndexOf('/') === dir.length - 1 && dir.length > 0 + ? dir + : `${dir}/`; + + const data = await getClient().listObjects({ + Bucket: bucket, + Prefix: normalisedPath + }); + const files = + data.Contents?.map(content => content.Key).filter( + key => key != undefined + ) ?? []; + + return Promise.all(files.map(file => removeFile(file))); + }; + + return removeFromContainer(); + }; + + const removeFile = async (filePath: string) => { + const client = getClient(); + + await client.deleteObject({ + Bucket: bucket, + Key: filePath + }); + }; + return { getFile, getJson, @@ -268,6 +304,8 @@ export default function s3Adapter(conf: S3Config): StorageAdapter { putDir, putFile, putFileContent, + removeFile, + removeDir, adapterType: 's3', isValid }; diff --git a/packages/oc-storage-adapters-utils/src/index.ts b/packages/oc-storage-adapters-utils/src/index.ts index ade0cec..c0fb6c0 100644 --- a/packages/oc-storage-adapters-utils/src/index.ts +++ b/packages/oc-storage-adapters-utils/src/index.ts @@ -30,5 +30,7 @@ export interface StorageAdapter { isPrivate: boolean, client?: unknown ): Promise; + removeDir(folderPath: string): Promise; + removeFile(filePath: string, isPrivate: boolean): Promise; isValid: () => boolean; }