From 6818e6960a2a04c10f96596f2f1b55ca6dea29a2 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Mon, 9 Nov 2020 10:36:36 +0100 Subject: [PATCH] feat: add support for connection string - NestJS 7 support - Upgrade Azure blob storage - Add support for connection string access Closes #7 #129 #127 --- lib/azure-storage.service.ts | 61 ++++++++++++++++------ schematics/install/schema.json | 15 +++--- schematics/install/schema.ts | 9 ++-- schematics/install/src/add-env-config.js | 42 +++++++++++---- schematics/install/src/add-env-config.ts | 66 +++++++++++++++++------- schematics/install/src/add-module.js | 4 +- schematics/install/src/add-module.ts | 10 ++-- 7 files changed, 143 insertions(+), 64 deletions(-) diff --git a/lib/azure-storage.service.ts b/lib/azure-storage.service.ts index 25816673..d770cef0 100644 --- a/lib/azure-storage.service.ts +++ b/lib/azure-storage.service.ts @@ -1,5 +1,5 @@ -import { ServiceClientOptions } from '@azure/ms-rest-js'; import { AbortController } from '@azure/abort-controller'; +import { ServiceClientOptions } from '@azure/ms-rest-js'; import { AnonymousCredential, BlobServiceClient } from '@azure/storage-blob'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { AZURE_STORAGE_MODULE_OPTIONS } from './azure-storage.constant'; @@ -9,7 +9,7 @@ export const APP_NAME = 'AzureStorageService'; export interface AzureStorageOptions { accountName: string; containerName: string; - sasKey?: string; + accessKey: string; clientOptions?: ServiceClientOptions; } @@ -46,9 +46,9 @@ export class AzureStorageService { ); } - if (!perRequestOptions.sasKey) { + if (!perRequestOptions.accessKey) { throw new Error( - `Error encountered: "AZURE_STORAGE_SAS_KEY" was not provided.`, + `Error encountered: "AZURE_STORAGE_ACCESS_KEY" was not provided.`, ); } @@ -60,13 +60,43 @@ export class AzureStorageService { ); } - const url = this.getServiceUrl(perRequestOptions); - const anonymousCredential = new AnonymousCredential(); - const blobServiceClient: BlobServiceClient = new BlobServiceClient( - // When using AnonymousCredential, following url should include a valid SAS or support public access - url, - anonymousCredential, - ); + // detect access type + let accessType: 'sasKey' | 'connectionString' = null; + if (perRequestOptions.accessKey.startsWith('BlobEndpoint')) { + accessType = 'connectionString'; + } else { + accessType = 'sasKey'; + } + + let blobServiceClient = null; + + // connection string + if (accessType === 'connectionString') { + // Create Blob Service Client from Account connection string or SAS connection string + // Account connection string example - `DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=accountKey;EndpointSuffix=core.windows.net` + // SAS connection string example - `BlobEndpoint=https://myaccount.blob.core.windows.net/;...;SharedAccessSignature=sasString` + blobServiceClient = BlobServiceClient.fromConnectionString( + perRequestOptions.accessKey, + ); + } else if (accessType === 'sasKey') { + // SAS key + // remove the first ? symbol if present + perRequestOptions.accessKey = perRequestOptions.accessKey.replace( + '?', + '', + ); + const url = this.getServiceUrl(perRequestOptions); + const anonymousCredential = new AnonymousCredential(); + blobServiceClient = new BlobServiceClient( + // When using AnonymousCredential, following url should include a valid SAS or support public access + url, + anonymousCredential, + ); + } else { + throw new Error( + `Error encountered: Connection string or SAS Token are missing`, + ); + } const containerClient = blobServiceClient.getContainerClient( perRequestOptions.containerName, @@ -135,9 +165,7 @@ export class AzureStorageService { } getServiceUrl(perRequestOptions: Partial) { - // remove the first ? symbol if present - perRequestOptions.sasKey = perRequestOptions.sasKey.replace('?', ''); - return `https://${perRequestOptions.accountName}.blob.core.windows.net/?${perRequestOptions.sasKey}`; + return `https://${perRequestOptions.accountName}.blob.core.windows.net/?${perRequestOptions.accessKey}`; } private async _listContainers(blobServiceClient: BlobServiceClient) { @@ -148,7 +176,10 @@ export class AzureStorageService { return containers; } - private async _doesContainerExist(blobServiceClient: BlobServiceClient, name: string) { + private async _doesContainerExist( + blobServiceClient: BlobServiceClient, + name: string, + ) { return (await this._listContainers(blobServiceClient)).includes(name); } } diff --git a/schematics/install/schema.json b/schematics/install/schema.json index c9415ea6..2b639722 100644 --- a/schematics/install/schema.json +++ b/schematics/install/schema.json @@ -31,18 +31,15 @@ "default": false }, "storageAccountName": { - "description": "The Azure Storage account name (see: http://bit.ly/azure-storage-account).", + "description": "The Azure storage account name (see: https://aka.ms/nestjs-azure-storage-account).", "type": "string", - "x-prompt": { - "message": "What is your Azure Storage account name (see: http://bit.ly/azure-storage-account)?", - "type": "string" - } + "x-prompt": "Your Azure storage account name" }, - "storageAccountSAS": { - "description": "The Azure Storage SAS Key (see: http://bit.ly/azure-storage-sas-key).", + "storageAccountAccessKey": { + "description": "The Azure storage account SAS token or connection string (see: https://aka.ms/nestjs-azure-storage-connection-string).", "type": "string", - "x-prompt": "What is your Azure Storage SAS key (see: http://bit.ly/azure-storage-sas-key)?" + "x-prompt": "The Azure storage account SAS token or connection string" } }, - "required": ["storageAccountName", "storageAccountSAS"] + "required": ["storageAccountName", "storageAccountAccessKey"] } diff --git a/schematics/install/schema.ts b/schematics/install/schema.ts index bf83b602..ac2a3f66 100644 --- a/schematics/install/schema.ts +++ b/schematics/install/schema.ts @@ -26,7 +26,10 @@ export interface Schema { storageAccountName: string; /** - * The Azure Storage SAS Key. + * The Azure Storage access tokens: + * - Connection string (SAS connection string or account connection string). + * - SAS token. */ - storageAccountSAS: string; -} + storageAccountAccessKey?: string; + storageAccountAccessType?: 'SASToken' | 'connectionString'; +} \ No newline at end of file diff --git a/schematics/install/src/add-env-config.js b/schematics/install/src/add-env-config.js index 973f004c..1ced65d1 100644 --- a/schematics/install/src/add-env-config.js +++ b/schematics/install/src/add-env-config.js @@ -4,28 +4,48 @@ exports.addDotEnvCall = exports.addDotEnvConfig = void 0; const core_1 = require("@angular-devkit/core"); const terminal_1 = require("@angular-devkit/core/src/terminal"); const schematics_1 = require("@angular-devkit/schematics"); -const AZURE_STORAGE_SAS_KEY = 'AZURE_STORAGE_SAS_KEY'; const AZURE_STORAGE_ACCOUNT = 'AZURE_STORAGE_ACCOUNT'; +const AZURE_STORAGE_ACCESS_KEY = 'AZURE_STORAGE_ACCESS_KEY'; function addDotEnvConfig(options) { return (tree, context) => { const envPath = core_1.normalize('/.env'); - if (options.storageAccountName === '' || options.storageAccountSAS === '') { + if (options.storageAccountName === '' || + options.storageAccountAccess === '') { if (options.storageAccountName === '') { - context.logger.error('storageAccountName can not be empty.'); + context.logger.error('The Azure storage account name can not be empty.'); } - if (options.storageAccountSAS === '') { - context.logger.error('storageAccountSAS can not be empty.'); + if (options.storageAccountAccess === '') { + context.logger.error('The Azure storage account SAS token (or connection string) can not be empty. ' + + 'Read more about how to generate an access token: https://aka.ms/nestjs-azure-storage-connection-strin'); } process.exit(1); return null; } - const newEnvFileContent = `# See: http://bit.ly/azure-storage-sas-key\n` + - `AZURE_STORAGE_SAS_KEY="${options.storageAccountSAS}"\n` + - `# See: http://bit.ly/azure-storage-account\n` + - `AZURE_STORAGE_ACCOUNT="${options.storageAccountName}"\n`; + if (options.storageAccountAccess.startsWith('BlobEndpoint')) { + options.storageAccountAccessType = 'connectionString'; + } + else if (options.storageAccountAccess.startsWith('?sv=') || + options.storageAccountAccess.startsWith('sv=')) { + options.storageAccountAccessType = 'SASToken'; + } + else { + context.logger.error('The Azure storage access key must be either a SAS token or a connection string. ' + + 'Read more: https://aka.ms/nestjs-azure-storage-connection-string'); + process.exit(1); + return null; + } + const newEnvFileContent = `# For more information about storage account: https://aka.ms/nestjs-azure-storage-account\n` + + `${AZURE_STORAGE_ACCOUNT}="${options.storageAccountName}"\n` + + `# For more information about storage access authorization: https://aka.ms/nestjs-azure-storage-connection-string\n` + + `${AZURE_STORAGE_ACCESS_KEY}="${options.storageAccountAccess}"\n`; const oldEnvFileContent = readEnvFile(tree, envPath); if (!oldEnvFileContent) { - tree.create(envPath, newEnvFileContent); + if (tree.exists(envPath)) { + tree.overwrite(envPath, newEnvFileContent); + } + else { + tree.create(envPath, newEnvFileContent); + } return tree; } if (oldEnvFileContent === newEnvFileContent) { @@ -33,7 +53,7 @@ function addDotEnvConfig(options) { `because an ".env" file was detected and already contains these Azure Storage tokens:\n\n` + terminal_1.green(`# New configuration\n` + `${newEnvFileContent}`)); } - if (oldEnvFileContent.includes(AZURE_STORAGE_SAS_KEY) || + if (oldEnvFileContent.includes(AZURE_STORAGE_ACCESS_KEY) || oldEnvFileContent.includes(AZURE_STORAGE_ACCOUNT)) { return context.logger.warn(`Skipping environment variables configuration ` + `because an ".env" file was detected and already contains an Azure Storage tokens.\n` + diff --git a/schematics/install/src/add-env-config.ts b/schematics/install/src/add-env-config.ts index 9bf7ec6c..8a638e9e 100644 --- a/schematics/install/src/add-env-config.ts +++ b/schematics/install/src/add-env-config.ts @@ -8,48 +8,76 @@ import { } from '@angular-devkit/schematics'; import { Schema as AzureOptions } from '../schema'; -const AZURE_STORAGE_SAS_KEY = 'AZURE_STORAGE_SAS_KEY'; const AZURE_STORAGE_ACCOUNT = 'AZURE_STORAGE_ACCOUNT'; - +const AZURE_STORAGE_ACCESS_KEY = 'AZURE_STORAGE_ACCESS_KEY'; /** - * This will create or update the `.env` file with the `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_SAS_KEY` values. - * + * This will create or update the `.env` file with the `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_ACCESS_KEY` values. + * * @example * ``` - * AZURE_STORAGE_SAS_KEY=this-is-the-sas-key-value - * AZURE_STORAGE_ACCOUNT=this-is-the-storage-account-value + * AZURE_STORAGE_ACCOUNT="storage-account-value" + * AZURE_STORAGE_ACCESS_KEY="sas-token-value-OR-connection-string" * ``` - * + * * @param options The Azure arguments provided to this schematic. */ export function addDotEnvConfig(options: AzureOptions): Rule { return (tree: Tree, context: SchematicContext) => { const envPath = normalize('/.env'); - - if (options.storageAccountName === '' || options.storageAccountSAS === '') { + if ( + options.storageAccountName === '' || + options.storageAccountAccessKey === '' + ) { if (options.storageAccountName === '') { - context.logger.error('storageAccountName can not be empty.') + context.logger.error( + 'The Azure storage account name can not be empty.', + ); } - if (options.storageAccountSAS === '') { - context.logger.error('storageAccountSAS can not be empty.') + if (options.storageAccountAccessKey === '') { + context.logger.error( + 'The Azure storage account SAS token (or connection string) can not be empty. ' + + 'Read more about how to generate an access token: https://aka.ms/nestjs-azure-storage-connection-string', + ); } process.exit(1); return null; } + if (options.storageAccountAccessKey.startsWith('BlobEndpoint')) { + options.storageAccountAccessType = 'connectionString'; + } else if ( + options.storageAccountAccessKey.startsWith('?sv=') || + options.storageAccountAccessKey.startsWith('sv=') + ) { + options.storageAccountAccessType = 'SASToken'; + } else { + context.logger.error( + 'The Azure storage access key must be either a SAS token or a connection string. ' + + 'Read more: https://aka.ms/nestjs-azure-storage-connection-string', + ); + process.exit(1); + return null; + } + // environment vars to add to .env file const newEnvFileContent = - `# See: http://bit.ly/azure-storage-sas-key\n` + - `AZURE_STORAGE_SAS_KEY="${options.storageAccountSAS}"\n` + - `# See: http://bit.ly/azure-storage-account\n` + - `AZURE_STORAGE_ACCOUNT="${options.storageAccountName}"\n`; + `# For more information about storage account: https://aka.ms/nestjs-azure-storage-account\n` + + `${AZURE_STORAGE_ACCOUNT}="${options.storageAccountName}"\n` + + `# For more information about storage access authorization: https://aka.ms/nestjs-azure-storage-connection-string\n` + + `${AZURE_STORAGE_ACCESS_KEY}="${options.storageAccountAccessKey}"\n`; const oldEnvFileContent = readEnvFile(tree, envPath); // .env file doest not exist, add one and exit if (!oldEnvFileContent) { - tree.create(envPath, newEnvFileContent); + if (tree.exists(envPath)) { + // file exists by empty + tree.overwrite(envPath, newEnvFileContent); + } else { + // file does not exists + tree.create(envPath, newEnvFileContent); + } return tree; } @@ -64,7 +92,7 @@ export function addDotEnvConfig(options: AzureOptions): Rule { // if old config was detected, verify config does not already contain the required tokens // otherwise we exit and let the user update manually their config if ( - oldEnvFileContent.includes(AZURE_STORAGE_SAS_KEY) || + oldEnvFileContent.includes(AZURE_STORAGE_ACCESS_KEY) || oldEnvFileContent.includes(AZURE_STORAGE_ACCOUNT) ) { return context.logger.warn( @@ -91,7 +119,7 @@ function readEnvFile(host: Tree, fileName: string): string { /** * This rule is responsible for adding the `require('dotenv').config()` to the main.ts file. * The call will be added to the top of the file before any other call. - * + * * @example * ``` * if (process.env.NODE_ENV !== 'production') require('dotenv').config(); diff --git a/schematics/install/src/add-module.js b/schematics/install/src/add-module.js index 17bd684e..c608eeba 100644 --- a/schematics/install/src/add-module.js +++ b/schematics/install/src/add-module.js @@ -6,10 +6,10 @@ const ast_1 = require("../utils/ast"); const nest_module_import_1 = require("../utils/nest-module-import"); function addAzureStorageModuleToImports(options) { return (tree, context) => { - const MODULE_WITH_CONFIG = `AzureStorageModule.withConfig({sasKey: process.env['AZURE_STORAGE_SAS_KEY'], accountName: process.env['AZURE_STORAGE_ACCOUNT'], containerName: 'nest-demo-container' })`; + const MODULE_WITH_CONFIG = `AzureStorageModule.withConfig({accessKey: process.env['AZURE_STORAGE_ACCESS_KEY'], accountName: process.env['AZURE_STORAGE_ACCOUNT'], containerName: 'nest-demo-container' })`; const appModulePath = core_1.normalize(options.rootDir + `/` + options.rootModuleFileName + `.ts`); if (nest_module_import_1.hasNestModuleImport(tree, appModulePath, MODULE_WITH_CONFIG)) { - return context.logger.warn(`>> Skiping importing "AzureStorageModule.withConfig()" because it is already imported in "${appModulePath}".`); + return context.logger.warn(`>> Skipping importing "AzureStorageModule.withConfig()" because it is already imported in "${appModulePath}".`); } ast_1.addModuleImportToRootModule(options)(tree, MODULE_WITH_CONFIG, '@nestjs/azure-storage'); return tree; diff --git a/schematics/install/src/add-module.ts b/schematics/install/src/add-module.ts index 0e16659b..8c94c4b6 100644 --- a/schematics/install/src/add-module.ts +++ b/schematics/install/src/add-module.ts @@ -28,9 +28,9 @@ import { hasNestModuleImport } from '../utils/nest-module-import'; * ``` * imports: [ * AzureStorageModule.withConfig({ - * sasKey: process.env['AZURE_STORAGE_SAS_KEY'], - * accountName: process.env['AZURE_STORAGE_ACCOUNT'], - * containerName: 'nest-demo-container', + * accessKey: process.env['AZURE_STORAGE_ACCESS_KEY'], + * accountName: process.env['AZURE_STORAGE_ACCOUNT'], + * containerName: 'nest-demo-container', * }), * ], * ``` @@ -39,7 +39,7 @@ import { hasNestModuleImport } from '../utils/nest-module-import'; */ export function addAzureStorageModuleToImports(options: AzureOptions): Rule { return (tree: Tree, context: SchematicContext) => { - const MODULE_WITH_CONFIG = `AzureStorageModule.withConfig({sasKey: process.env['AZURE_STORAGE_SAS_KEY'], accountName: process.env['AZURE_STORAGE_ACCOUNT'], containerName: 'nest-demo-container' })`; + const MODULE_WITH_CONFIG = `AzureStorageModule.withConfig({accessKey: process.env['AZURE_STORAGE_ACCESS_KEY'], accountName: process.env['AZURE_STORAGE_ACCOUNT'], containerName: 'nest-demo-container' })`; const appModulePath = normalize( options.rootDir + `/` + options.rootModuleFileName + `.ts`, ); @@ -47,7 +47,7 @@ export function addAzureStorageModuleToImports(options: AzureOptions): Rule { // verify module has not already been imported if (hasNestModuleImport(tree, appModulePath, MODULE_WITH_CONFIG)) { return context.logger.warn( - `>> Skiping importing "AzureStorageModule.withConfig()" because it is already imported in "${appModulePath}".`, + `>> Skipping importing "AzureStorageModule.withConfig()" because it is already imported in "${appModulePath}".`, ); }