From 0fe00c491a4a737bde0bd719adbc3bbc70f90617 Mon Sep 17 00:00:00 2001 From: Erin Beal Date: Thu, 10 Oct 2024 09:51:55 -0700 Subject: [PATCH 01/20] feat: add baseEndpoint to advanced options --- .../storage/src/internals/types/inputs.ts | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index ac807801627..2855a2411c3 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -44,14 +44,17 @@ export interface GetDataAccessInput { scope: string; } +export interface AdvancedOptions { + locationCredentialsProvider?: CredentialsProvider; + baseEndpoint?: string; +} + /** * @internal */ export type ListInputWithPath = ExtendInputWithAdvancedOptions< ListAllWithPathInput | ListPaginateWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -59,9 +62,7 @@ export type ListInputWithPath = ExtendInputWithAdvancedOptions< */ export type RemoveInput = ExtendInputWithAdvancedOptions< RemoveWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -69,9 +70,7 @@ export type RemoveInput = ExtendInputWithAdvancedOptions< */ export type GetPropertiesInput = ExtendInputWithAdvancedOptions< GetPropertiesWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -79,9 +78,7 @@ export type GetPropertiesInput = ExtendInputWithAdvancedOptions< */ export type GetUrlInput = ExtendInputWithAdvancedOptions< GetUrlWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -89,9 +86,7 @@ export type GetUrlInput = ExtendInputWithAdvancedOptions< */ export type CopyInput = ExtendCopyInputWithAdvancedOptions< CopyWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -99,9 +94,7 @@ export type CopyInput = ExtendCopyInputWithAdvancedOptions< */ export type DownloadDataInput = ExtendInputWithAdvancedOptions< DownloadDataWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** From a3c264ce4adccf22efccbf33f9d3f7b43730755f Mon Sep 17 00:00:00 2001 From: Erin Beal Date: Thu, 10 Oct 2024 11:57:51 -0700 Subject: [PATCH 02/20] feat: add baseEndpoint to customEndpoint --- .../providers/s3/utils/client/s3data/base.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index c7aef5c033c..4a18fe8dfcb 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -32,6 +32,7 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { * Whether to use the S3 Transfer Acceleration endpoint. */ useAccelerateEndpoint?: boolean; + /** * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region or * useAccelerateEndpoint config. @@ -39,6 +40,11 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ customEndpoint?: string; + /** + * Prepends a custom endpoint with information dependent on endpoint type (e.g., https://[account-id].s3-control-fips.us-west-2.amazonaws.com) + */ + baseEndpoint?: string; + /** * Whether to force path style URLs for S3 objects (e.g., https://s3.amazonaws.com// instead of * https://.s3.amazonaws.com/ @@ -54,12 +60,23 @@ const endpointResolver = ( options: S3EndpointResolverOptions, apiInput?: { Bucket?: string }, ) => { - const { region, useAccelerateEndpoint, customEndpoint, forcePathStyle } = - options; + const { + region, + useAccelerateEndpoint, + customEndpoint, + baseEndpoint, + forcePathStyle, + } = options; let endpoint: URL; // 1. get base endpoint if (customEndpoint) { - endpoint = new AmplifyUrl(customEndpoint); + if (baseEndpoint) { + endpoint = new AmplifyUrl( + `https://${baseEndpoint}.${customEndpoint.replace('https://', '')}`, + ); + } else { + endpoint = new AmplifyUrl(customEndpoint); + } } else if (useAccelerateEndpoint) { if (forcePathStyle) { throw new Error( From 9807007be024e2d950552b0b29e9d34fcb2164c1 Mon Sep 17 00:00:00 2001 From: Erin Beal Date: Thu, 10 Oct 2024 14:16:25 -0700 Subject: [PATCH 03/20] feat: thread baseEndpoint through resolved config to endpoint resolver --- packages/storage/src/providers/s3/types/options.ts | 1 + .../storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index badb9267d39..676b70e0c58 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -270,6 +270,7 @@ export interface CopyWithPathDestinationOptions { export interface ResolvedS3Config extends Pick { customEndpoint?: string; + baseEndpoint?: string; forcePathStyle?: boolean; useAccelerateEndpoint?: boolean; } diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index f19be5ddca7..0f38aa645c4 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -29,6 +29,7 @@ interface S3ApiOptions { targetIdentityId?: string; useAccelerateEndpoint?: boolean; locationCredentialsProvider?: LocationCredentialsProvider; + baseEndpoint?: string; bucket?: StorageBucket; } @@ -133,6 +134,7 @@ export const resolveS3ConfigAndInput = async ( credentials: credentialsProvider, region, useAccelerateEndpoint: apiOptions?.useAccelerateEndpoint, + baseEndpoint: apiOptions?.baseEndpoint, ...(dangerouslyConnectToHttpEndpointForTesting ? { customEndpoint: LOCAL_TESTING_S3_ENDPOINT, From 4b26c1fb82f4a6b27fd2ae79da80d6d7bc09e28d Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 28 Oct 2024 22:46:42 -0700 Subject: [PATCH 04/20] add customEndpoint advanced option to internals storage data-plane apis --- packages/storage/src/internals/apis/copy.ts | 1 + packages/storage/src/internals/apis/downloadData.ts | 1 + packages/storage/src/internals/apis/getProperties.ts | 1 + packages/storage/src/internals/apis/getUrl.ts | 1 + packages/storage/src/internals/apis/list.ts | 1 + packages/storage/src/internals/apis/remove.ts | 1 + packages/storage/src/internals/apis/uploadData.ts | 1 + packages/storage/src/internals/types/inputs.ts | 6 ++---- packages/storage/src/providers/s3/types/options.ts | 1 - 9 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/storage/src/internals/apis/copy.ts b/packages/storage/src/internals/apis/copy.ts index eb53241e89e..3286ab99462 100644 --- a/packages/storage/src/internals/apis/copy.ts +++ b/packages/storage/src/internals/apis/copy.ts @@ -27,6 +27,7 @@ export const copy = (input: CopyInput) => options: { // Advanced options locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `copyInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts index 8f097f82bb6..bd862d9d9b4 100644 --- a/packages/storage/src/internals/apis/downloadData.ts +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -18,6 +18,7 @@ export const downloadData = (input: DownloadDataInput): DownloadDataOutput => bytesRange: input?.options?.bytesRange, onProgress: input?.options?.onProgress, expectedBucketOwner: input?.options?.expectedBucketOwner, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `downloadDataInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/getProperties.ts b/packages/storage/src/internals/apis/getProperties.ts index 54e03c6d60b..213e184edae 100644 --- a/packages/storage/src/internals/apis/getProperties.ts +++ b/packages/storage/src/internals/apis/getProperties.ts @@ -20,6 +20,7 @@ export const getProperties = ( bucket: input?.options?.bucket, locationCredentialsProvider: input?.options?.locationCredentialsProvider, expectedBucketOwner: input?.options?.expectedBucketOwner, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/getUrl.ts b/packages/storage/src/internals/apis/getUrl.ts index 3c19b922e17..ef82f107c67 100644 --- a/packages/storage/src/internals/apis/getUrl.ts +++ b/packages/storage/src/internals/apis/getUrl.ts @@ -24,6 +24,7 @@ export const getUrl = (input: GetUrlInput) => // Advanced options locationCredentialsProvider: input?.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/list.ts b/packages/storage/src/internals/apis/list.ts index 4ce0a72eb3f..60c9184bd7f 100644 --- a/packages/storage/src/internals/apis/list.ts +++ b/packages/storage/src/internals/apis/list.ts @@ -39,6 +39,7 @@ export function list(input: ListInput): Promise { pageSize: (input as ListPaginateInput).options?.pageSize, // Advanced options locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `listInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/remove.ts b/packages/storage/src/internals/apis/remove.ts index 22864c3156b..96530325e2c 100644 --- a/packages/storage/src/internals/apis/remove.ts +++ b/packages/storage/src/internals/apis/remove.ts @@ -18,6 +18,7 @@ export const remove = (input: RemoveInput): Promise => bucket: input?.options?.bucket, expectedBucketOwner: input?.options?.expectedBucketOwner, locationCredentialsProvider: input?.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `removeInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/uploadData.ts b/packages/storage/src/internals/apis/uploadData.ts index 58b04ac2211..1ebabdb6165 100644 --- a/packages/storage/src/internals/apis/uploadData.ts +++ b/packages/storage/src/internals/apis/uploadData.ts @@ -27,6 +27,7 @@ export const uploadData = (input: UploadDataInput) => { // Advanced options locationCredentialsProvider: options?.locationCredentialsProvider, + customEndpoint: options?.customEndpoint, }, // Type casting is necessary because `uploadDataInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index 2d3833cec79..50f6ca06272 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -45,7 +45,7 @@ export interface GetDataAccessInput { export interface AdvancedOptions { locationCredentialsProvider?: CredentialsProvider; - baseEndpoint?: string; + customEndpoint?: string; } /** @@ -103,9 +103,7 @@ export type CopyInput = ExtendCopyInputWithAdvancedOptions< export type UploadDataInput = ExtendInputWithAdvancedOptions< UploadDataWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index b157300ac3e..39891185185 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -280,7 +280,6 @@ export interface CopyWithPathDestinationOptions { export interface ResolvedS3Config extends Pick { customEndpoint?: string; - baseEndpoint?: string; forcePathStyle?: boolean; useAccelerateEndpoint?: boolean; } From 3a898133e505ddb37406c49042057b85809105e0 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 00:55:21 -0700 Subject: [PATCH 05/20] add customEndpoint advanced option to internals storage control-plane apis --- .../src/internals/apis/getDataAccess.ts | 1 + .../internals/apis/listCallerAccessGrants.ts | 21 ++++++++++++++- .../storage/src/internals/types/inputs.ts | 7 +++-- .../s3/utils/client/s3control/base.ts | 5 +++- .../providers/s3/utils/client/s3data/base.ts | 27 +++++++------------ .../s3/utils/resolveS3ConfigAndInput.ts | 6 +++-- 6 files changed, 43 insertions(+), 24 deletions(-) diff --git a/packages/storage/src/internals/apis/getDataAccess.ts b/packages/storage/src/internals/apis/getDataAccess.ts index 3a6af14441a..070bf617078 100644 --- a/packages/storage/src/internals/apis/getDataAccess.ts +++ b/packages/storage/src/internals/apis/getDataAccess.ts @@ -33,6 +33,7 @@ export const getDataAccess = async ( const result = await getDataAccessClient( { credentials: clientCredentialsProvider, + customEndpoint: input.customEndpoint, region: input.region, userAgentValue: getStorageUserAgentValue(StorageAction.GetDataAccess), }, diff --git a/packages/storage/src/internals/apis/listCallerAccessGrants.ts b/packages/storage/src/internals/apis/listCallerAccessGrants.ts index c0da06b4f93..e9414d72b8d 100644 --- a/packages/storage/src/internals/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/internals/apis/listCallerAccessGrants.ts @@ -20,7 +20,14 @@ import { MAX_PAGE_SIZE } from '../utils/constants'; export const listCallerAccessGrants = async ( input: ListCallerAccessGrantsInput, ): Promise => { - const { credentialsProvider, accountId, region, nextToken, pageSize } = input; + const { + credentialsProvider, + accountId, + region, + nextToken, + pageSize, + customEndpoint, + } = input; logger.debug(`listing available locations from account ${input.accountId}`); @@ -40,6 +47,7 @@ export const listCallerAccessGrants = async ( await listCallerAccessGrantsClient( { credentials: clientCredentialsProvider, + customEndpoint, region, userAgentValue: getStorageUserAgentValue( StorageAction.ListCallerAccessGrants, @@ -94,3 +102,14 @@ function assertGrantScope(value: unknown): asserts value is string { }); } } + +// 1. Default +// // `${bucket}.s3.${region}.amazonaws.com`; + +// 2. AccelerateEndpoint +// // `${bucket}.s3-accelerate.${region}.amazonaws.com`; + +// 3. AccelerateEndpoint +// // `${bucket}.s3-accelerate.${region}.amazonaws.com`; + +// `https://${AmplifyResolvedBucket}.${CustomEndpointBucket}.s3.${CustomEndpointRegion}.amazonaws.com`; diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index 50f6ca06272..e2fefca539e 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -23,7 +23,9 @@ import { Permission, PrefixType, Privilege } from './common'; /** * @internal */ -export interface ListCallerAccessGrantsInput extends ListLocationsInput { +export interface ListCallerAccessGrantsInput + extends ListLocationsInput, + Pick { accountId: string; credentialsProvider: CredentialsProvider; region: string; @@ -32,7 +34,8 @@ export interface ListCallerAccessGrantsInput extends ListLocationsInput { /** * @internal */ -export interface GetDataAccessInput { +export interface GetDataAccessInput + extends Pick { accountId: string; credentialsProvider: CredentialsProvider; durationSeconds?: number; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index 590f2b26120..7164af8f0dc 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -42,7 +42,10 @@ const endpointResolver = ( let endpoint: URL; // 1. get base endpoint if (customEndpoint) { - endpoint = new AmplifyUrl(customEndpoint); + if (customEndpoint.includes('://')) { + throw new Error('Invalid S3 Endpoint.'); + } + endpoint = new AmplifyUrl(`https://${customEndpoint}`); } else if (accountId) { // Control plane operations endpoint = new AmplifyUrl( diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 4a18fe8dfcb..73bfbd7cdc2 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -12,6 +12,7 @@ import { } from '@aws-amplify/core/internals/aws-client-utils'; import { createRetryDecider, createXmlErrorParser } from '../utils'; +import { LOCAL_TESTING_S3_ENDPOINT } from '../../constants'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -40,11 +41,6 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ customEndpoint?: string; - /** - * Prepends a custom endpoint with information dependent on endpoint type (e.g., https://[account-id].s3-control-fips.us-west-2.amazonaws.com) - */ - baseEndpoint?: string; - /** * Whether to force path style URLs for S3 objects (e.g., https://s3.amazonaws.com// instead of * https://.s3.amazonaws.com/ @@ -52,7 +48,6 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ forcePathStyle?: boolean; }; - /** * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters. */ @@ -60,22 +55,18 @@ const endpointResolver = ( options: S3EndpointResolverOptions, apiInput?: { Bucket?: string }, ) => { - const { - region, - useAccelerateEndpoint, - customEndpoint, - baseEndpoint, - forcePathStyle, - } = options; + const { region, useAccelerateEndpoint, customEndpoint, forcePathStyle } = + options; let endpoint: URL; // 1. get base endpoint if (customEndpoint) { - if (baseEndpoint) { - endpoint = new AmplifyUrl( - `https://${baseEndpoint}.${customEndpoint.replace('https://', '')}`, - ); - } else { + if (customEndpoint === LOCAL_TESTING_S3_ENDPOINT) { endpoint = new AmplifyUrl(customEndpoint); + } else { + if (customEndpoint.includes('://')) { + throw new Error('Invalid S3 Endpoint.'); + } + endpoint = new AmplifyUrl(`https://${customEndpoint}`); } } else if (useAccelerateEndpoint) { if (forcePathStyle) { diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index 0f38aa645c4..7cb4c55316e 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -29,7 +29,7 @@ interface S3ApiOptions { targetIdentityId?: string; useAccelerateEndpoint?: boolean; locationCredentialsProvider?: LocationCredentialsProvider; - baseEndpoint?: string; + customEndpoint?: string; bucket?: StorageBucket; } @@ -134,7 +134,9 @@ export const resolveS3ConfigAndInput = async ( credentials: credentialsProvider, region, useAccelerateEndpoint: apiOptions?.useAccelerateEndpoint, - baseEndpoint: apiOptions?.baseEndpoint, + ...(apiOptions?.customEndpoint + ? { customEndpoint: apiOptions.customEndpoint } + : {}), ...(dangerouslyConnectToHttpEndpointForTesting ? { customEndpoint: LOCAL_TESTING_S3_ENDPOINT, From 86e7c8b177e60332c61672e056e30d77ffb6a3c0 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 01:20:38 -0700 Subject: [PATCH 06/20] fix unit test --- .../providers/s3/utils/client/S3/cases/getObject.ts | 2 +- .../src/internals/apis/listCallerAccessGrants.ts | 11 ----------- .../src/providers/s3/utils/client/s3data/base.ts | 3 +-- scripts/dts-bundler/s3-control.d.ts | 4 ++-- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index a35c813f3d8..d6f4ec6f189 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -176,7 +176,7 @@ const getObjectCustomEndpoint: ApiFunctionalTestCase = [ getObject, { ...defaultConfig, - customEndpoint: 'https://custom.endpoint.com', + customEndpoint: 'custom.endpoint.com', forcePathStyle: true, } as Parameters[0], { diff --git a/packages/storage/src/internals/apis/listCallerAccessGrants.ts b/packages/storage/src/internals/apis/listCallerAccessGrants.ts index e9414d72b8d..47fee1c051a 100644 --- a/packages/storage/src/internals/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/internals/apis/listCallerAccessGrants.ts @@ -102,14 +102,3 @@ function assertGrantScope(value: unknown): asserts value is string { }); } } - -// 1. Default -// // `${bucket}.s3.${region}.amazonaws.com`; - -// 2. AccelerateEndpoint -// // `${bucket}.s3-accelerate.${region}.amazonaws.com`; - -// 3. AccelerateEndpoint -// // `${bucket}.s3-accelerate.${region}.amazonaws.com`; - -// `https://${AmplifyResolvedBucket}.${CustomEndpointBucket}.s3.${CustomEndpointRegion}.amazonaws.com`; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 73bfbd7cdc2..7db6a7826db 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -33,14 +33,12 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { * Whether to use the S3 Transfer Acceleration endpoint. */ useAccelerateEndpoint?: boolean; - /** * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region or * useAccelerateEndpoint config. * The path of this endpoint */ customEndpoint?: string; - /** * Whether to force path style URLs for S3 objects (e.g., https://s3.amazonaws.com// instead of * https://.s3.amazonaws.com/ @@ -48,6 +46,7 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ forcePathStyle?: boolean; }; + /** * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters. */ diff --git a/scripts/dts-bundler/s3-control.d.ts b/scripts/dts-bundler/s3-control.d.ts index 1c5443611a7..e6d727c5fba 100644 --- a/scripts/dts-bundler/s3-control.d.ts +++ b/scripts/dts-bundler/s3-control.d.ts @@ -1,8 +1,8 @@ import { - GetDataAccessCommandInput, - GetDataAccessCommandOutput, ListCallerAccessGrantsCommandInput, ListCallerAccessGrantsCommandOutput, + GetDataAccessCommandInput, + GetDataAccessCommandOutput, } from '@aws-sdk/client-s3-control'; export { From fa5bf950db207afc53746f6d7ec73167ac762cc3 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 01:21:29 -0700 Subject: [PATCH 07/20] code cleanup --- packages/core/src/parseAmplifyOutputs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/parseAmplifyOutputs.ts b/packages/core/src/parseAmplifyOutputs.ts index 51e20bf98e6..21e52752812 100644 --- a/packages/core/src/parseAmplifyOutputs.ts +++ b/packages/core/src/parseAmplifyOutputs.ts @@ -5,6 +5,7 @@ /* eslint-disable camelcase */ /* Does not like exhaustive checks */ +/* eslint-disable no-case-declarations */ import { APIConfig, From 40563b76159bc2442c94afc98e4d103c61218f3c Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 01:31:06 -0700 Subject: [PATCH 08/20] increase bundle size --- packages/aws-amplify/package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 30a90ed816d..e685dc96db8 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,37 +461,37 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.05 kB" + "limit": "16.10 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "16.40 kB" + "limit": "16.46 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.65 kB" + "limit": "15.69 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.90 kB" + "limit": "16.94 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "16.35 kB" + "limit": "16.39 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.50 kB" + "limit": "15.56 kB" }, { "name": "[Storage] uploadData (S3)", From d00a5607b36d88a45f23d8be372f8ef06a9c518b Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 17:26:54 -0700 Subject: [PATCH 09/20] wire up customEndpoint on copy API --- packages/storage/src/providers/s3/apis/internal/copy.ts | 1 + .../storage/src/providers/s3/utils/client/s3control/base.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 5098096a81f..85898ea228f 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -80,6 +80,7 @@ const copyWithPath = async ( path: input.destination.path, options: { locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input.options?.customEndpoint, ...input.destination, }, }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index 7164af8f0dc..b2d553a2269 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -45,7 +45,7 @@ const endpointResolver = ( if (customEndpoint.includes('://')) { throw new Error('Invalid S3 Endpoint.'); } - endpoint = new AmplifyUrl(`https://${customEndpoint}`); + endpoint = new AmplifyUrl(`https://${accountId}.${customEndpoint}`); } else if (accountId) { // Control plane operations endpoint = new AmplifyUrl( From 5d643ae9c8a3762434fabd4dbc5080363032993e Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 29 Oct 2024 17:36:50 -0700 Subject: [PATCH 10/20] increase the bundle size --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index e685dc96db8..9cd4e7e85d0 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,7 +461,7 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.10 kB" + "limit": "16.12 kB" }, { "name": "[Storage] downloadData (S3)", From 906b4e10b99f1b377bef05edfcd2da205ce7c9c5 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Thu, 31 Oct 2024 16:43:48 -0700 Subject: [PATCH 11/20] add customEndpoint unit tests for all data and control apis --- .../client/S3/cases/abortMultipartUpload.ts | 36 ++++++++++++++- .../S3/cases/completeMultipartUpload.ts | 43 +++++++++++++++++- .../s3/utils/client/S3/cases/copyObject.ts | 32 ++++++++++++- .../client/S3/cases/createMultipartUpload.ts | 32 ++++++++++++- .../s3/utils/client/S3/cases/deleteObject.ts | 32 ++++++++++++- .../s3/utils/client/S3/cases/getDataAccess.ts | 38 +++++++++++++++- .../s3/utils/client/S3/cases/getObject.ts | 12 +++-- .../s3/utils/client/S3/cases/headObject.ts | 32 ++++++++++++- .../client/S3/cases/listCallerAccessGrants.ts | 32 +++++++++++++ .../s3/utils/client/S3/cases/listObjectsV2.ts | 45 +++++++++++++++++++ .../s3/utils/client/S3/cases/listParts.ts | 33 +++++++++++++- .../s3/utils/client/S3/cases/putObject.ts | 33 +++++++++++++- .../s3/utils/client/S3/cases/uploadPart.ts | 34 +++++++++++++- 13 files changed, 418 insertions(+), 16 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts index cc81a2be88f..d9cd460b659 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts @@ -42,4 +42,38 @@ const abortMultipartUploadHappyCase: ApiFunctionalTestCase< }, ]; -export default [abortMultipartUploadHappyCase]; +const abortMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof abortMultipartUpload +> = [ + 'happy case', + 'abortMultipartUpload with custom endpoint', + abortMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 204, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [ + abortMultipartUploadHappyCase, + abortMultipartUploadHappyCaseCustomEndpoint, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 74a914767c4..94c6af39983 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -81,7 +81,6 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< }, ]; -// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase< typeof completeMultipartUpload > = [ @@ -104,7 +103,46 @@ const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase< completeMultipartUploadHappyCase[7], ]; -// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html +const completeMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof completeMultipartUpload +> = [ + 'happy case', + 'completeMultipartUpload with custom endpoint', + completeMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + MultipartUpload: { + Parts: [ + { + ETag: 'etag1', + PartNumber: 1, + ChecksumCRC32: 'test-checksum-1', + }, + ], + }, + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const completeMultipartUploadErrorCase: ApiFunctionalTestCase< typeof completeMultipartUpload > = [ @@ -167,6 +205,7 @@ const completeMultipartUploadErrorWith200CodeCase: ApiFunctionalTestCase< export default [ completeMultipartUploadHappyCase, completeMultipartUploadHappyCaseIfNoneMatch, + completeMultipartUploadHappyCaseCustomEndpoint, completeMultipartUploadErrorCase, completeMultipartUploadErrorWith200CodeCase, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts index a1a52247846..e1ffebe127d 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts @@ -58,4 +58,34 @@ const copyObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [copyObjectHappyCase]; +const copyObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof copyObject +> = [ + 'happy case', + 'getObject with custom endpoint', + copyObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + CopySource: 'sourceBucket/sourceKey', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; +export default [copyObjectHappyCase, copyObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts index e027397e569..afc83f772a6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts @@ -42,4 +42,34 @@ const createMultiPartUploadHappyCase: ApiFunctionalTestCase< }, ]; -export default [createMultiPartUploadHappyCase]; +const createMultiPartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof createMultipartUpload +> = [ + 'happy case', + 'createMultipartUpload with custom endpoint', + createMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + putObjectRequest, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploads', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [ + createMultiPartUploadHappyCase, + createMultiPartUploadHappyCaseCustomEndpoint, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts index 614a3c1fff6..46cede55900 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts @@ -36,4 +36,34 @@ const deleteObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [deleteObjectHappyCase]; +const deleteObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof deleteObject +> = [ + 'happy case', + 'deleteObject with custom endpoint', + deleteObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [deleteObjectHappyCase, deleteObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts index 851bc993a7c..9d25d8ff91c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -71,6 +71,38 @@ const getDataAccessHappyCase: ApiFunctionalTestCase = [ }, ]; +const getDataAccessHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof getDataAccess +> = [ + 'happy case', + 'getDataAccess with custom endpoint', + getDataAccess, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + } as Parameters[0], + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + Permission: 'READWRITE', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/dataaccess?permission=READWRITE&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const getDataAccessErrorCase: ApiFunctionalTestCase = [ 'error case', 'getDataAccess', @@ -99,4 +131,8 @@ const getDataAccessErrorCase: ApiFunctionalTestCase = [ }, ]; -export default [getDataAccessHappyCase, getDataAccessErrorCase]; +export default [ + getDataAccessHappyCase, + getDataAccessHappyCaseCustomEndpoint, + getDataAccessErrorCase, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index d6f4ec6f189..acafd19babc 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -143,7 +143,9 @@ const getObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -const getObjectAccelerateEndpoint: ApiFunctionalTestCase = [ +const getObjectHappyCaseAccelerateEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ 'happy case', 'getObject with accelerate endpoint', getObject, @@ -170,7 +172,9 @@ const getObjectAccelerateEndpoint: ApiFunctionalTestCase = [ }) as any, ]; -const getObjectCustomEndpoint: ApiFunctionalTestCase = [ +const getObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ 'happy case', 'getObject with custom endpoint', getObject, @@ -200,6 +204,6 @@ const getObjectCustomEndpoint: ApiFunctionalTestCase = [ export default [ getObjectHappyCase, - getObjectAccelerateEndpoint, - getObjectCustomEndpoint, + getObjectHappyCaseAccelerateEndpoint, + getObjectHappyCaseCustomEndpoint, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts index 0cc016a7813..654531fbb28 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts @@ -48,4 +48,34 @@ const headObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [headObjectHappyCase]; +const headObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof headObject +> = [ + 'happy case', + 'headObject with custom endpoint', + headObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [headObjectHappyCase, headObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index 63499b7234c..e20386c198b 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -137,6 +137,37 @@ const listCallerAccessGrantsHappyCaseMultipleGrants: ApiFunctionalTestCase< }, ]; +const listCallerAccessGrantsHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrants with custom endpoint', + listCallerAccessGrants, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + } as Parameters[0], + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< typeof listCallerAccessGrants > = [ @@ -170,5 +201,6 @@ const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< export default [ listCallerAccessGrantsHappyCaseSingleGrant, listCallerAccessGrantsHappyCaseMultipleGrants, + listCallerAccessGrantsHappyCaseCustomEndpoint, listCallerAccessGrantsErrorCase, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index a14ec159fa3..bd9ad59bc19 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -376,9 +376,54 @@ const listObjectsV2ErrorCaseMissingTruncated: ApiFunctionalTestCase< }, ]; +const listObjectsV2HappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'happy case', + 'listObjectsV2 with custom endpoint', + listObjectsV2, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Prefix: 'Prefix', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket?list-type=2&prefix=Prefix', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + bucket + + 1 + 1000 + false + + ExampleObject.txt + 2013-09-17T18:07:53.000Z + "599bab3ed2c697f1d26842727561fd94" + 857 + REDUCED_REDUNDANCY + + `, + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + export default [ listObjectsV2HappyCaseTruncated, listObjectsV2HappyCaseComplete, + listObjectsV2HappyCaseCustomEndpoint, listObjectsV2ErrorCaseKeyCount, listObjectsV2ErrorCaseMissingTruncated, listObjectsV2ErrorCaseMissingToken, diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts index 059dfcaec0a..a2020fee93a 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts @@ -66,4 +66,35 @@ const listPartsHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [listPartsHappyCase]; +const listPartsHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listParts +> = [ + 'happy case', + 'listParts with custom endpoint', + listParts, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [listPartsHappyCase, listPartsHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts index 867ee3f0af2..3f7e9a7c361 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts @@ -68,7 +68,32 @@ const putObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -const pubObjectDefaultContentType: ApiFunctionalTestCase = [ +const putObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof putObject +> = [ + 'happy case', + 'putObject with custom endpoint', + putObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + putObjectRequest, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + putObjectSuccessResponse, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +const pubObjectHappyCaseDefaultContentType: ApiFunctionalTestCase< + typeof putObject +> = [ 'happy case', 'putObject default content type', putObject, @@ -86,4 +111,8 @@ const pubObjectDefaultContentType: ApiFunctionalTestCase = [ expect.anything(), ]; -export default [putObjectHappyCase, pubObjectDefaultContentType]; +export default [ + putObjectHappyCase, + putObjectHappyCaseCustomEndpoint, + pubObjectHappyCaseDefaultContentType, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts index 4a46891c849..8a9ef5f4838 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts @@ -44,4 +44,36 @@ const uploadPartHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [uploadPartHappyCase]; +const uploadPartHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof uploadPart +> = [ + 'happy case', + 'uploadPart with custom endpoint', + uploadPart, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + } as Parameters[0], + { + Bucket: 'bucket', + Key: 'key', + PartNumber: 1, + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?partNumber=1&uploadId=uploadId', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS, etag: 'etag' }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [uploadPartHappyCase, uploadPartHappyCaseCustomEndpoint]; From 9e8bde1f3dd0e51597c5ee5f458061e4ab866e75 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Thu, 31 Oct 2024 17:00:11 -0700 Subject: [PATCH 12/20] increase bundle size --- packages/aws-amplify/package.json | 14 +++++++------- .../src/providers/s3/utils/client/s3data/base.ts | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 2a808815c19..2430832bc3f 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.15 kB" + "limit": "16.19 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "16.48 kB" + "limit": "16.54 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.72 kB" + "limit": "15.76 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.98 kB" + "limit": "17.03 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "16.45 kB" + "limit": "16.49 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.59 kB" + "limit": "15.64 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.56 kB" + "limit": "22.592 kB" } ] } diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 7db6a7826db..15e96923843 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -61,10 +61,9 @@ const endpointResolver = ( if (customEndpoint) { if (customEndpoint === LOCAL_TESTING_S3_ENDPOINT) { endpoint = new AmplifyUrl(customEndpoint); + } else if (customEndpoint.includes('://')) { + throw new Error('Invalid S3 Endpoint.'); } else { - if (customEndpoint.includes('://')) { - throw new Error('Invalid S3 Endpoint.'); - } endpoint = new AmplifyUrl(`https://${customEndpoint}`); } } else if (useAccelerateEndpoint) { From e1a5a075b08091364ec6a46e60c05e50a7f21bd2 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 00:45:12 -0800 Subject: [PATCH 13/20] update ts docs --- .../s3/utils/client/s3control/base.ts | 28 +++++++++++++------ .../providers/s3/utils/client/s3data/base.ts | 17 +++++++++-- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index b2d553a2269..91b63c56dcc 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -26,6 +26,21 @@ export const SERVICE_NAME = 's3'; export type S3EndpointResolverOptions = EndpointResolverOptions & { /** * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region. + * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region + * + * A fully qualified custom endpoint for S3. If set, this endpoint will override + * the default S3 control endpoint and be used regardless of the specified region configuration. + * + * Refer to AWS documentation for more details on available endpoints: + * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region + * + * @example + * ```ts + * // Examples of S3 custom endpoints + * const endpoint1 = "s3-control.us-east-2.amazonaws.com"; + * const endpoint2 = "s3-control.dualstack.us-east-2.amazonaws.com"; + * const endpoint3 = "s3-control-fips.dualstack.us-east-2.amazonaws.com"; + * ``` */ customEndpoint?: string; }; @@ -35,25 +50,20 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ const endpointResolver = ( options: S3EndpointResolverOptions, - apiInput?: { AccountId?: string }, + apiInput?: { AccountId: string }, ) => { const { region, customEndpoint } = options; - const { AccountId: accountId } = apiInput || {}; + const { AccountId: accountId } = apiInput ?? {}; let endpoint: URL; - // 1. get base endpoint + if (customEndpoint) { if (customEndpoint.includes('://')) { throw new Error('Invalid S3 Endpoint.'); } endpoint = new AmplifyUrl(`https://${accountId}.${customEndpoint}`); - } else if (accountId) { - // Control plane operations - endpoint = new AmplifyUrl( - `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, - ); } else { endpoint = new AmplifyUrl( - `https://s3-control.${region}.${getDnsSuffix(region)}`, + `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, ); } diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 15e96923843..08afdfb3380 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -34,9 +34,20 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ useAccelerateEndpoint?: boolean; /** - * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region or - * useAccelerateEndpoint config. - * The path of this endpoint + * A fully qualified custom endpoint for S3. If set, this endpoint will override + * the default S3 endpoint and be used regardless of the specified region or + * `useAccelerateEndpoint` configuration. + * + * Refer to AWS documentation for more details on available endpoints: + * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region + * + * @example + * ```ts + * // Examples of S3 custom endpoints + * const endpoint1 = "s3.us-east-2.amazonaws.com"; + * const endpoint2 = "s3.dualstack.us-east-2.amazonaws.com"; + * const endpoint3 = "s3-fips.dualstack.us-east-2.amazonaws.com"; + * ``` */ customEndpoint?: string; /** From c79b4a165fae4b2980f50f314877af113214e40f Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 15:17:26 -0800 Subject: [PATCH 14/20] add additional error unit tests for endpointResolver --- .../utils/resolveS3ConfigAndInput.test.ts | 2 +- .../client/S3/cases/abortMultipartUpload.ts | 2 +- .../S3/cases/completeMultipartUpload.ts | 2 +- .../s3/utils/client/S3/cases/copyObject.ts | 2 +- .../client/S3/cases/createMultipartUpload.ts | 2 +- .../s3/utils/client/S3/cases/deleteObject.ts | 2 +- .../s3/utils/client/S3/cases/getDataAccess.ts | 36 ++++++- .../s3/utils/client/S3/cases/getObject.ts | 96 ++++++++++++++++++- .../s3/utils/client/S3/cases/headObject.ts | 2 +- .../client/S3/cases/listCallerAccessGrants.ts | 2 +- .../s3/utils/client/S3/cases/listObjectsV2.ts | 2 +- .../s3/utils/client/S3/cases/listParts.ts | 2 +- .../s3/utils/client/S3/cases/putObject.ts | 2 +- .../s3/utils/client/S3/cases/uploadPart.ts | 2 +- .../storage/src/errors/types/validation.ts | 18 +++- .../s3/utils/client/s3control/base.ts | 11 ++- .../providers/s3/utils/client/s3data/base.ts | 28 +++--- .../s3/utils/resolveS3ConfigAndInput.ts | 2 +- 18 files changed, 181 insertions(+), 34 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index 662640e3340..3a90ba5fff6 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -335,7 +335,7 @@ describe('resolveS3ConfigAndInput', () => { }); } catch (error: any) { expect(error).toBeInstanceOf(StorageError); - expect(error.name).toBe(StorageValidationErrorCode.InvalidStorageBucket); + expect(error.name).toBe(StorageValidationErrorCode.StorageBucketNotFound); } }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts index d9cd460b659..5eba75535a6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts @@ -52,7 +52,7 @@ const abortMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 94c6af39983..140267b751b 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -113,7 +113,7 @@ const completeMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts index e1ffebe127d..c20c0394a0e 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts @@ -68,7 +68,7 @@ const copyObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts index afc83f772a6..b53ae0b48e8 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts @@ -52,7 +52,7 @@ const createMultiPartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, putObjectRequest, expect.objectContaining({ url: expect.objectContaining({ diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts index 46cede55900..0d591b6bfcc 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts @@ -46,7 +46,7 @@ const deleteObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts index 9d25d8ff91c..6e944a058a6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -80,7 +80,7 @@ const getDataAccessHappyCaseCustomEndpoint: ApiFunctionalTestCase< { ...defaultConfig, customEndpoint: 'custom.endpoint.com', - } as Parameters[0], + }, { AccountId: MOCK_ACCOUNT_ID, Target: 's3://my-bucket/path/to/object.md', @@ -131,8 +131,42 @@ const getDataAccessErrorCase: ApiFunctionalTestCase = [ }, ]; +const getDataAccessErrorCaseInvalidCustomEndpoint: ApiFunctionalTestCase< + typeof getDataAccess +> = [ + 'error case', + 'getDataAccess with invalid custom endpoint', + getDataAccess, + { + ...defaultConfig, + customEndpoint: 'http://custom.endpoint.com', + }, + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + Permission: 'READWRITE', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/dataaccess?permission=READWRITE&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + { + message: 'Invalid S3 custom endpoint.', + name: 'InvalidCustomEndpoint', + }, +]; + export default [ getDataAccessHappyCase, getDataAccessHappyCaseCustomEndpoint, getDataAccessErrorCase, + getDataAccessErrorCaseInvalidCustomEndpoint, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index acafd19babc..7034207002d 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -152,7 +152,7 @@ const getObjectHappyCaseAccelerateEndpoint: ApiFunctionalTestCase< { ...defaultConfig, useAccelerateEndpoint: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', @@ -182,7 +182,7 @@ const getObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', @@ -202,8 +202,100 @@ const getObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< }) as any, ]; +const getObjectErrorCaseAccelerateEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with accelerate endpoint and forcePathStyle', + getObject, + { + ...defaultConfig, + useAccelerateEndpoint: true, + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://bucket.s3-accelerate.amazonaws.com/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: 'Path style URLs are not supported with S3 Transfer Acceleration.', + name: 'ForcePathStyleEndpointNotSupported', + }, +]; + +const getObjectErrorCaseInvalidCustomEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with invalid custom endpoint', + getObject, + { + ...defaultConfig, + customEndpoint: 'http://custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: 'Invalid S3 custom endpoint.', + name: 'InvalidCustomEndpoint', + }, +]; + +const getObjectErrorCaseInvalidBucketName: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with incompatible Dns bucket name', + getObject, + defaultConfig, + { + Bucket: 'incompatibleDnsCompatibleBucketName', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://incompatibleDnsCompatibleBucketName.s3.us-east-1.amazonaws.com/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: 'Invalid bucket name', + name: 'InvalidStorageBucketName', + }, +]; + export default [ getObjectHappyCase, getObjectHappyCaseAccelerateEndpoint, getObjectHappyCaseCustomEndpoint, + getObjectErrorCaseAccelerateEndpoint, + getObjectErrorCaseInvalidCustomEndpoint, + getObjectErrorCaseInvalidBucketName, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts index 654531fbb28..a392e121c8c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts @@ -58,7 +58,7 @@ const headObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index e20386c198b..175e6d8b0da 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -146,7 +146,7 @@ const listCallerAccessGrantsHappyCaseCustomEndpoint: ApiFunctionalTestCase< { ...defaultConfig, customEndpoint: 'custom.endpoint.com', - } as Parameters[0], + }, { AccountId: MOCK_ACCOUNT_ID, GrantScope: 's3://my-bucket/path/to/', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index bd9ad59bc19..aa60b74f906 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -386,7 +386,7 @@ const listObjectsV2HappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Prefix: 'Prefix', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts index a2020fee93a..63f2a37e06c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts @@ -76,7 +76,7 @@ const listPartsHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts index 3f7e9a7c361..6ee6f1e62fa 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts @@ -78,7 +78,7 @@ const putObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, putObjectRequest, expect.objectContaining({ url: expect.objectContaining({ diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts index 8a9ef5f4838..34d0d6f7f38 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts @@ -54,7 +54,7 @@ const uploadPartHappyCaseCustomEndpoint: ApiFunctionalTestCase< ...defaultConfig, customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index de15d0a89ec..64170ba750b 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -13,7 +13,8 @@ export enum StorageValidationErrorCode { NoDestinationPath = 'NoDestinationPath', NoBucket = 'NoBucket', NoRegion = 'NoRegion', - InvalidStorageBucket = 'InvalidStorageBucket', + StorageBucketNotFound = 'StorageBucketNotFound', + InvalidStorageBucketName = 'InvalidStorageBucketName', InvalidCopyOperationStorageBucket = 'InvalidCopyOperationStorageBucket', InvalidStorageOperationPrefixInput = 'InvalidStorageOperationPrefixInput', InvalidStorageOperationInput = 'InvalidStorageOperationInput', @@ -25,6 +26,8 @@ export enum StorageValidationErrorCode { InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', InvalidS3Uri = 'InvalidS3Uri', + InvalidCustomEndpoint = 'InvalidCustomEndpoint', + ForcePathStyleEndpointNotSupported = 'ForcePathStyleEndpointNotSupported', } export const validationErrorMap: AmplifyErrorMap = { @@ -88,11 +91,22 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.InvalidS3Uri]: { message: 'Invalid S3 URI.', }, - [StorageValidationErrorCode.InvalidStorageBucket]: { + [StorageValidationErrorCode.InvalidStorageBucketName]: { + message: 'Invalid bucket name', + }, + [StorageValidationErrorCode.StorageBucketNotFound]: { message: 'Unable to lookup bucket from provided name in Amplify configuration.', }, [StorageValidationErrorCode.InvalidCopyOperationStorageBucket]: { message: 'Missing bucket option in either source or destination.', }, + [StorageValidationErrorCode.InvalidCustomEndpoint]: { + message: 'Invalid S3 custom endpoint.', + recoverySuggestion: + 'Refer to AWS documentation for more details on available endpoints: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region', + }, + [StorageValidationErrorCode.ForcePathStyleEndpointNotSupported]: { + message: 'Path style URLs are not supported with S3 Transfer Acceleration.', + }, }; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index 91b63c56dcc..455c9ac4cbc 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -12,6 +12,8 @@ import { } from '@aws-amplify/core/internals/aws-client-utils'; import { createRetryDecider, createXmlErrorParser } from '../utils'; +import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; /** * The service name used to sign requests if the API requires authentication. @@ -26,7 +28,6 @@ export const SERVICE_NAME = 's3'; export type S3EndpointResolverOptions = EndpointResolverOptions & { /** * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region. - * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region * * A fully qualified custom endpoint for S3. If set, this endpoint will override * the default S3 control endpoint and be used regardless of the specified region configuration. @@ -53,13 +54,15 @@ const endpointResolver = ( apiInput?: { AccountId: string }, ) => { const { region, customEndpoint } = options; + // TODO(ashwinkumar6): make accountId a required param const { AccountId: accountId } = apiInput ?? {}; let endpoint: URL; if (customEndpoint) { - if (customEndpoint.includes('://')) { - throw new Error('Invalid S3 Endpoint.'); - } + assertValidationError( + !customEndpoint.includes('://'), + StorageValidationErrorCode.InvalidCustomEndpoint, + ); endpoint = new AmplifyUrl(`https://${accountId}.${customEndpoint}`); } else { endpoint = new AmplifyUrl( diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 08afdfb3380..e1cc91d6ea5 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -13,6 +13,8 @@ import { import { createRetryDecider, createXmlErrorParser } from '../utils'; import { LOCAL_TESTING_S3_ENDPOINT } from '../../constants'; +import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -72,26 +74,28 @@ const endpointResolver = ( if (customEndpoint) { if (customEndpoint === LOCAL_TESTING_S3_ENDPOINT) { endpoint = new AmplifyUrl(customEndpoint); - } else if (customEndpoint.includes('://')) { - throw new Error('Invalid S3 Endpoint.'); - } else { - endpoint = new AmplifyUrl(`https://${customEndpoint}`); } + assertValidationError( + !customEndpoint.includes('://'), + StorageValidationErrorCode.InvalidCustomEndpoint, + ); + endpoint = new AmplifyUrl(`https://${customEndpoint}`); } else if (useAccelerateEndpoint) { - if (forcePathStyle) { - throw new Error( - 'Path style URLs are not supported with S3 Transfer Acceleration.', - ); - } + assertValidationError( + !forcePathStyle, + StorageValidationErrorCode.ForcePathStyleEndpointNotSupported, + ); endpoint = new AmplifyUrl(`https://s3-accelerate.${getDnsSuffix(region)}`); } else { endpoint = new AmplifyUrl(`https://s3.${region}.${getDnsSuffix(region)}`); } // 2. inject bucket name if (apiInput?.Bucket) { - if (!isDnsCompatibleBucketName(apiInput.Bucket)) { - throw new Error(`Invalid bucket name: "${apiInput.Bucket}".`); - } + assertValidationError( + isDnsCompatibleBucketName(apiInput.Bucket), + StorageValidationErrorCode.InvalidStorageBucketName, + ); + if (forcePathStyle || apiInput.Bucket.includes('.')) { endpoint.pathname = `/${apiInput.Bucket}`; } else { diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index 7cb4c55316e..9e4f50cfbfd 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -218,7 +218,7 @@ const resolveBucketConfig = ( const bucketConfig = buckets?.[apiOptions.bucket]; assertValidationError( !!bucketConfig, - StorageValidationErrorCode.InvalidStorageBucket, + StorageValidationErrorCode.StorageBucketNotFound, ); return { bucket: bucketConfig.bucketName, region: bucketConfig.region }; From a836cd8635a84107ac6deb047b746585c8e4c3a2 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 16:47:06 -0800 Subject: [PATCH 15/20] add unit tests for internals/ apis --- packages/aws-amplify/package.json | 14 +++++++------- .../storage/__tests__/internals/apis/copy.test.ts | 2 ++ .../__tests__/internals/apis/downloadData.test.ts | 3 +++ .../__tests__/internals/apis/getDataAccess.test.ts | 4 +++- .../__tests__/internals/apis/getProperties.test.ts | 3 +++ .../__tests__/internals/apis/getUrl.test.ts | 3 +++ .../storage/__tests__/internals/apis/list.test.ts | 3 +++ .../internals/apis/listCallerAccessGrants.test.ts | 7 +++++-- .../__tests__/internals/apis/remove.test.ts | 3 +++ .../__tests__/internals/apis/uploadData.test.ts | 4 ++++ 10 files changed, 36 insertions(+), 10 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 2430832bc3f..538941d4fa7 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.19 kB" + "limit": "16.32 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "16.54 kB" + "limit": "16.66 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.76 kB" + "limit": "15.92 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "17.03 kB" + "limit": "17.15 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "16.49 kB" + "limit": "16.62 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.64 kB" + "limit": "15.76 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.592 kB" + "limit": "22.73 kB" } ] } diff --git a/packages/storage/__tests__/internals/apis/copy.test.ts b/packages/storage/__tests__/internals/apis/copy.test.ts index f7fc957cae7..2692f4f6a68 100644 --- a/packages/storage/__tests__/internals/apis/copy.test.ts +++ b/packages/storage/__tests__/internals/apis/copy.test.ts @@ -17,6 +17,7 @@ describe('copy (internals)', () => { }); it('should pass advanced option locationCredentialsProvider to internal list', async () => { + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -40,6 +41,7 @@ describe('copy (internals)', () => { }, options: { locationCredentialsProvider, + customEndpoint, }, }; const result = await advancedCopy(copyInputWithAdvancedOptions); diff --git a/packages/storage/__tests__/internals/apis/downloadData.test.ts b/packages/storage/__tests__/internals/apis/downloadData.test.ts index 6175e91e7d0..f18ea441e69 100644 --- a/packages/storage/__tests__/internals/apis/downloadData.test.ts +++ b/packages/storage/__tests__/internals/apis/downloadData.test.ts @@ -31,6 +31,7 @@ describe('downloadData (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -45,6 +46,7 @@ describe('downloadData (internal)', () => { const output = await advancedDownloadData({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, @@ -58,6 +60,7 @@ describe('downloadData (internal)', () => { expect(mockedDownloadDataInternal).toHaveBeenCalledWith({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, diff --git a/packages/storage/__tests__/internals/apis/getDataAccess.test.ts b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts index 630e7835f9e..34c41fe2bc7 100644 --- a/packages/storage/__tests__/internals/apis/getDataAccess.test.ts +++ b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts @@ -31,10 +31,11 @@ const MOCK_ACCESS_CREDENTIALS = { SessionToken: MOCK_SESSION_TOKEN, Expiration: MOCK_EXPIRATION_DATE, }; +const MOCK_CUSTOM_ENDPOINT = 's3-accesspoint.dualstack.us-east-2.amazonaws.com'; const MOCK_CREDENTIAL_PROVIDER = jest.fn().mockResolvedValue(MOCK_CREDENTIALS); - const sharedGetDataAccessParams: GetDataAccessInput = { accountId: MOCK_ACCOUNT_ID, + customEndpoint: MOCK_CUSTOM_ENDPOINT, credentialsProvider: MOCK_CREDENTIAL_PROVIDER, durationSeconds: 900, permission: 'READWRITE', @@ -62,6 +63,7 @@ describe('getDataAccess', () => { expect(getDataAccessClientMock).toHaveBeenCalledWith( expect.objectContaining({ credentials: expect.any(Function), + customEndpoint: MOCK_CUSTOM_ENDPOINT, region: MOCK_REGION, userAgentValue: expect.stringContaining('storage/8'), }), diff --git a/packages/storage/__tests__/internals/apis/getProperties.test.ts b/packages/storage/__tests__/internals/apis/getProperties.test.ts index 97e85210e36..aa0c2c9815e 100644 --- a/packages/storage/__tests__/internals/apis/getProperties.test.ts +++ b/packages/storage/__tests__/internals/apis/getProperties.test.ts @@ -23,6 +23,7 @@ describe('getProperties (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -34,6 +35,7 @@ describe('getProperties (internal)', () => { const result = await advancedGetProperties({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -46,6 +48,7 @@ describe('getProperties (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/getUrl.test.ts b/packages/storage/__tests__/internals/apis/getUrl.test.ts index d1b83149b7a..fcffafd3f2e 100644 --- a/packages/storage/__tests__/internals/apis/getUrl.test.ts +++ b/packages/storage/__tests__/internals/apis/getUrl.test.ts @@ -32,6 +32,7 @@ describe('getUrl (internal)', () => { const contentDisposition = 'inline; filename="example.jpg"'; const contentType = 'image/jpeg'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -43,6 +44,7 @@ describe('getUrl (internal)', () => { const result = await advancedGetUrl({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, validateObjectExistence, @@ -59,6 +61,7 @@ describe('getUrl (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, validateObjectExistence, diff --git a/packages/storage/__tests__/internals/apis/list.test.ts b/packages/storage/__tests__/internals/apis/list.test.ts index d72b95ad0bd..16ea0e5037b 100644 --- a/packages/storage/__tests__/internals/apis/list.test.ts +++ b/packages/storage/__tests__/internals/apis/list.test.ts @@ -20,6 +20,7 @@ describe('list (internals)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -31,6 +32,7 @@ describe('list (internals)', () => { const result = await advancedList({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -43,6 +45,7 @@ describe('list (internals)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts index ebe724e688c..e25931749b0 100644 --- a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts @@ -21,6 +21,7 @@ const mockCredentialsProvider = jest .mockResolvedValue({ credentials: mockCredentials }); const mockNextToken = '123'; const mockPageSize = 123; +const mockCustomEndpoint = 's3-accesspoint.dualstack.us-east-2.amazonaws.com'; describe('listCallerAccessGrants', () => { afterEach(() => { @@ -36,6 +37,7 @@ describe('listCallerAccessGrants', () => { }); await listCallerAccessGrants({ accountId: mockAccountId, + customEndpoint: mockCustomEndpoint, region: mockRegion, credentialsProvider: mockCredentialsProvider, nextToken: mockNextToken, @@ -45,6 +47,7 @@ describe('listCallerAccessGrants', () => { expect.objectContaining({ region: mockRegion, credentials: expect.any(Function), + customEndpoint: mockCustomEndpoint, }), expect.objectContaining({ AccountId: mockAccountId, @@ -66,7 +69,7 @@ describe('listCallerAccessGrants', () => { }); }); - it('should set a default page size', async () => { + it.skip('should set a default page size', async () => { expect.assertions(1); jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ NextToken: undefined, @@ -86,7 +89,7 @@ describe('listCallerAccessGrants', () => { ); }); - it('should set response location type correctly', async () => { + it.skip('should set response location type correctly', async () => { expect.assertions(2); jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ NextToken: undefined, diff --git a/packages/storage/__tests__/internals/apis/remove.test.ts b/packages/storage/__tests__/internals/apis/remove.test.ts index 2f67997e10c..2adab6dd0ef 100644 --- a/packages/storage/__tests__/internals/apis/remove.test.ts +++ b/packages/storage/__tests__/internals/apis/remove.test.ts @@ -23,6 +23,7 @@ describe('remove (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -35,6 +36,7 @@ describe('remove (internal)', () => { const result = await advancedRemove({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -48,6 +50,7 @@ describe('remove (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/uploadData.test.ts b/packages/storage/__tests__/internals/apis/uploadData.test.ts index 642e4776e74..e9345d12cc6 100644 --- a/packages/storage/__tests__/internals/apis/uploadData.test.ts +++ b/packages/storage/__tests__/internals/apis/uploadData.test.ts @@ -21,6 +21,8 @@ describe('uploadData (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; + const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -37,6 +39,7 @@ describe('uploadData (internal)', () => { path: 'input/path/to/mock/object', data: 'data', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, @@ -54,6 +57,7 @@ describe('uploadData (internal)', () => { path: 'input/path/to/mock/object', data: 'data', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, From dada6aeca795d218eea7b3921b051fb8219c97c9 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 16:57:37 -0800 Subject: [PATCH 16/20] code cleanup --- .../__tests__/internals/apis/listCallerAccessGrants.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts index e25931749b0..43d96f24488 100644 --- a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts @@ -69,7 +69,7 @@ describe('listCallerAccessGrants', () => { }); }); - it.skip('should set a default page size', async () => { + it('should set a default page size', async () => { expect.assertions(1); jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ NextToken: undefined, @@ -89,7 +89,7 @@ describe('listCallerAccessGrants', () => { ); }); - it.skip('should set response location type correctly', async () => { + it('should set response location type correctly', async () => { expect.assertions(2); jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ NextToken: undefined, From 1bea2c717bbd673ab4826e52646ead6969588bfc Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 18:13:51 -0800 Subject: [PATCH 17/20] address feedback --- .../s3/apis/utils/resolveS3ConfigAndInput.test.ts | 2 +- .../s3/utils/client/S3/cases/getObject.ts | 4 ++-- packages/storage/src/errors/types/validation.ts | 14 ++++++++------ .../src/providers/s3/utils/client/s3data/base.ts | 2 +- .../providers/s3/utils/resolveS3ConfigAndInput.ts | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index 3a90ba5fff6..662640e3340 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -335,7 +335,7 @@ describe('resolveS3ConfigAndInput', () => { }); } catch (error: any) { expect(error).toBeInstanceOf(StorageError); - expect(error.name).toBe(StorageValidationErrorCode.StorageBucketNotFound); + expect(error.name).toBe(StorageValidationErrorCode.InvalidStorageBucket); } }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index 7034207002d..2a2ddf98f68 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -286,8 +286,8 @@ const getObjectErrorCaseInvalidBucketName: ApiFunctionalTestCase< body: 'mockBody', }, { - message: 'Invalid bucket name', - name: 'InvalidStorageBucketName', + message: `The bucket name isn't DNS compatible.`, + name: 'DnsIncompatibleBucketName', }, ]; diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index 64170ba750b..aff2da5d51c 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -13,8 +13,7 @@ export enum StorageValidationErrorCode { NoDestinationPath = 'NoDestinationPath', NoBucket = 'NoBucket', NoRegion = 'NoRegion', - StorageBucketNotFound = 'StorageBucketNotFound', - InvalidStorageBucketName = 'InvalidStorageBucketName', + InvalidStorageBucket = 'InvalidStorageBucket', InvalidCopyOperationStorageBucket = 'InvalidCopyOperationStorageBucket', InvalidStorageOperationPrefixInput = 'InvalidStorageOperationPrefixInput', InvalidStorageOperationInput = 'InvalidStorageOperationInput', @@ -28,6 +27,7 @@ export enum StorageValidationErrorCode { InvalidS3Uri = 'InvalidS3Uri', InvalidCustomEndpoint = 'InvalidCustomEndpoint', ForcePathStyleEndpointNotSupported = 'ForcePathStyleEndpointNotSupported', + DnsIncompatibleBucketName = 'DnsIncompatibleBucketName', } export const validationErrorMap: AmplifyErrorMap = { @@ -91,10 +91,7 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.InvalidS3Uri]: { message: 'Invalid S3 URI.', }, - [StorageValidationErrorCode.InvalidStorageBucketName]: { - message: 'Invalid bucket name', - }, - [StorageValidationErrorCode.StorageBucketNotFound]: { + [StorageValidationErrorCode.InvalidStorageBucket]: { message: 'Unable to lookup bucket from provided name in Amplify configuration.', }, @@ -109,4 +106,9 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.ForcePathStyleEndpointNotSupported]: { message: 'Path style URLs are not supported with S3 Transfer Acceleration.', }, + [StorageValidationErrorCode.DnsIncompatibleBucketName]: { + message: `The bucket name isn't DNS compatible.`, + recoverySuggestion: + 'Refer to AWS documentation for buckets naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names', + }, }; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index e1cc91d6ea5..72c1e6c30cd 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -93,7 +93,7 @@ const endpointResolver = ( if (apiInput?.Bucket) { assertValidationError( isDnsCompatibleBucketName(apiInput.Bucket), - StorageValidationErrorCode.InvalidStorageBucketName, + StorageValidationErrorCode.DnsIncompatibleBucketName, ); if (forcePathStyle || apiInput.Bucket.includes('.')) { diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index 9e4f50cfbfd..7cb4c55316e 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -218,7 +218,7 @@ const resolveBucketConfig = ( const bucketConfig = buckets?.[apiOptions.bucket]; assertValidationError( !!bucketConfig, - StorageValidationErrorCode.StorageBucketNotFound, + StorageValidationErrorCode.InvalidStorageBucket, ); return { bucket: bucketConfig.bucketName, region: bucketConfig.region }; From 231fe546d591f8f10ba4d35514801e7bc9b91973 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 4 Nov 2024 18:17:04 -0800 Subject: [PATCH 18/20] add comment for ForcePathStyleEndpointNotSupported ErrorCode --- packages/storage/src/providers/s3/utils/client/s3data/base.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index 72c1e6c30cd..fdf6160d077 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -81,6 +81,7 @@ const endpointResolver = ( ); endpoint = new AmplifyUrl(`https://${customEndpoint}`); } else if (useAccelerateEndpoint) { + // this ErrorCode isn't expose yet since forcePathStyle param isn't publicly exposed assertValidationError( !forcePathStyle, StorageValidationErrorCode.ForcePathStyleEndpointNotSupported, From 5a2979a688552b8a935b3f7afc887e9cda665930 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 5 Nov 2024 01:37:01 -0800 Subject: [PATCH 19/20] increase bundle size --- packages/aws-amplify/package.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 538941d4fa7..9dab516b859 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.32 kB" + "limit": "16.39 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "16.66 kB" + "limit": "16.73 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.92 kB" + "limit": "15.99 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "17.15 kB" + "limit": "17.22 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "16.62 kB" + "limit": "16.69 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.76 kB" + "limit": "15.83 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.73 kB" + "limit": "22.81 kB" } ] } From 906f457626503dc4668bdc5bb9cea702ec6ee1a6 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 5 Nov 2024 10:16:24 -0800 Subject: [PATCH 20/20] remove docs links from error recovery message --- packages/storage/src/errors/types/validation.ts | 4 ---- packages/storage/src/internals/types/inputs.ts | 9 ++++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index aff2da5d51c..a56662adec4 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -100,15 +100,11 @@ export const validationErrorMap: AmplifyErrorMap = { }, [StorageValidationErrorCode.InvalidCustomEndpoint]: { message: 'Invalid S3 custom endpoint.', - recoverySuggestion: - 'Refer to AWS documentation for more details on available endpoints: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region', }, [StorageValidationErrorCode.ForcePathStyleEndpointNotSupported]: { message: 'Path style URLs are not supported with S3 Transfer Acceleration.', }, [StorageValidationErrorCode.DnsIncompatibleBucketName]: { message: `The bucket name isn't DNS compatible.`, - recoverySuggestion: - 'Refer to AWS documentation for buckets naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names', }, }; diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index e2fefca539e..a79d171ff08 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -23,21 +23,20 @@ import { Permission, PrefixType, Privilege } from './common'; /** * @internal */ -export interface ListCallerAccessGrantsInput - extends ListLocationsInput, - Pick { +export interface ListCallerAccessGrantsInput extends ListLocationsInput { accountId: string; credentialsProvider: CredentialsProvider; + customEndpoint?: string; region: string; } /** * @internal */ -export interface GetDataAccessInput - extends Pick { +export interface GetDataAccessInput { accountId: string; credentialsProvider: CredentialsProvider; + customEndpoint?: string; durationSeconds?: number; permission: Permission; prefixType?: PrefixType;