diff --git a/.changeset/rare-dodos-exist.md b/.changeset/rare-dodos-exist.md new file mode 100644 index 00000000000..41035d501f2 --- /dev/null +++ b/.changeset/rare-dodos-exist.md @@ -0,0 +1,5 @@ +--- +"@smithy/types": major +--- + +improved streaming payload types diff --git a/.gitignore b/.gitignore index b2ced78ad21..dba5e0e2d01 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ smithy-typescript-integ-tests/yarn.lock # Issue https://github.com/awslabs/smithy-typescript/issues/425 smithy-typescript-codegen/bin/ +smithy-typescript-ssdk-codegen-test-utils/bin/ **/node_modules/ **/*.tsbuildinfo diff --git a/packages/types/README.md b/packages/types/README.md index a42a15877e0..3075576a0ab 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -2,3 +2,40 @@ [![NPM version](https://img.shields.io/npm/v/@smithy/types/latest.svg)](https://www.npmjs.com/package/@smithy/types) [![NPM downloads](https://img.shields.io/npm/dm/@smithy/types.svg)](https://www.npmjs.com/package/@smithy/types) + +## Usage + +This package is mostly used internally by generated clients. +Some public components have independent applications. + +### Scenario: Narrowing a smithy-typescript generated client's output payload blob types + +--- + +This is mostly relevant to operations with streaming bodies such as within +the S3Client in the AWS SDK for JavaScript v3. + +Because blob payload types are platform dependent, you may wish to indicate in your application that a client is running in a specific +environment. This narrows the blob payload types. + +```typescript +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import type { NodeJsClient, SdkStream, StreamingBlobPayloadOutputTypes } from "@smithy/types"; +import type { IncomingMessage } from "node:http"; + +// default client init. +const s3Default = new S3Client({}); + +// client init with type narrowing. +const s3NarrowType = new S3Client({}) as NodeJsClient; + +// The default type of blob payloads is a wide union type including multiple possible +// request handlers. +const body1: StreamingBlobPayloadOutputTypes = (await s3Default.send(new GetObjectCommand({ Key: "", Bucket: "" }))) + .Body!; + +// This is of the narrower type SdkStream representing +// blob payload responses using specifically the node:http request handler. +const body2: SdkStream = (await s3NarrowType.send(new GetObjectCommand({ Key: "", Bucket: "" }))) + .Body!; +``` diff --git a/packages/types/src/blob/blob-payload-input-types.ts b/packages/types/src/blob/blob-payload-input-types.ts new file mode 100644 index 00000000000..5bb380ecca0 --- /dev/null +++ b/packages/types/src/blob/blob-payload-input-types.ts @@ -0,0 +1,48 @@ +import { Readable } from "stream"; + +/** + * @public + * + * A union of types that can be used as inputs for the service model + * "blob" type when it represents the request's entire payload or body. + * + * For example, in Lambda::invoke, the payload is modeled as a blob type + * and this union applies to it. + * In contrast, in Lambda::createFunction the Zip file option is a blob type, + * but is not the (entire) payload and this union does not apply. + * + * Note: not all types are signable by the standard SignatureV4 signer when + * used as the request body. For example, in Node.js a Readable stream + * is not signable by the default signer. + * They are included in the union because it may work in some cases, + * but the expected types are primarily string and Uint8Array. + * + * Additional details may be found in the internal + * function "getPayloadHash" in the SignatureV4 module. + */ +export type BlobPayloadInputTypes = + | string + | ArrayBuffer + | ArrayBufferView + | Uint8Array + | NodeJsRuntimeBlobTypes + | BrowserRuntimeBlobTypes; + +/** + * @public + * + * Additional blob types for the Node.js environment. + */ +export type NodeJsRuntimeBlobTypes = Readable | Buffer; + +/** + * @public + * + * Additional blob types for the browser environment. + */ +export type BrowserRuntimeBlobTypes = Blob | ReadableStream; + +/** + * @deprecated renamed to BlobPayloadInputTypes. + */ +export type BlobTypes = BlobPayloadInputTypes; diff --git a/packages/types/src/client.ts b/packages/types/src/client.ts index da86d4782de..cb92b23c4a8 100644 --- a/packages/types/src/client.ts +++ b/packages/types/src/client.ts @@ -7,11 +7,19 @@ import { MetadataBearer } from "./response"; * * function definition for different overrides of client's 'send' function. */ -interface InvokeFunction { +export interface InvokeFunction< + InputTypes extends object, + OutputTypes extends MetadataBearer, + ResolvedClientConfiguration +> { ( command: Command, options?: any ): Promise; + ( + command: Command, + cb: (err: any, data?: OutputType) => void + ): void; ( command: Command, options: any, @@ -24,6 +32,18 @@ interface InvokeFunction | void; } +/** + * @internal + * + * Signature that appears on aggregated clients' methods. + */ +export interface InvokeMethod { + (input: InputType, options?: any): Promise; + (input: InputType, cb: (err: any, data?: OutputType) => void): void; + (input: InputType, options: any, cb: (err: any, data?: OutputType) => void): void; + (input: InputType, options?: any, cb?: (err: any, data?: OutputType) => void): Promise | void; +} + /** * A general interface for service clients, idempotent to browser or node clients * This type corresponds to SmithyClient(https://github.com/aws/aws-sdk-js-v3/blob/main/packages/smithy-client/src/client.ts). diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 45415085d19..696f569ca1d 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -1,5 +1,6 @@ export * from "./abort"; export * from "./auth"; +export * from "./blob/blob-payload-input-types"; export * from "./checksum"; export * from "./client"; export * from "./command"; @@ -21,7 +22,12 @@ export * from "./serde"; export * from "./shapes"; export * from "./signature"; export * from "./stream"; +export * from "./streaming-payload/streaming-blob-common-types"; +export * from "./streaming-payload/streaming-blob-payload-input-types"; +export * from "./streaming-payload/streaming-blob-payload-output-types"; export * from "./transfer"; +export * from "./transform/client-payload-blob-type-narrow"; +export * from "./transform/type-transform"; export * from "./uri"; export * from "./util"; export * from "./waiter"; diff --git a/packages/types/src/streaming-payload/streaming-blob-common-types.ts b/packages/types/src/streaming-payload/streaming-blob-common-types.ts new file mode 100644 index 00000000000..c953a2f7636 --- /dev/null +++ b/packages/types/src/streaming-payload/streaming-blob-common-types.ts @@ -0,0 +1,34 @@ +import type { Readable } from "stream"; + +/** + * @public + * + * This is the union representing the modeled blob type with streaming trait + * in a generic format that does not relate to HTTP input or output payloads. + * + * Note: the non-streaming blob type is represented by Uint8Array, but because + * the streaming blob type is always in the request/response paylod, it has + * historically been handled with different types. + * + * @see https://smithy.io/2.0/spec/simple-types.html#blob + * + * For compatibility with its historical representation, it must contain at least + * Readble (Node.js), Blob (browser), and ReadableStream (browser). + * + * @see StreamingPayloadInputTypes for FAQ about mixing types from multiple environments. + */ +export type StreamingBlobTypes = NodeJsRuntimeStreamingBlobTypes | BrowserRuntimeStreamingBlobTypes; + +/** + * @public + * + * Node.js streaming blob type. + */ +export type NodeJsRuntimeStreamingBlobTypes = Readable; + +/** + * @public + * + * Browser streaming blob types. + */ +export type BrowserRuntimeStreamingBlobTypes = ReadableStream | Blob; diff --git a/packages/types/src/streaming-payload/streaming-blob-payload-input-types.ts b/packages/types/src/streaming-payload/streaming-blob-payload-input-types.ts new file mode 100644 index 00000000000..d9c7ef65e59 --- /dev/null +++ b/packages/types/src/streaming-payload/streaming-blob-payload-input-types.ts @@ -0,0 +1,64 @@ +import type { Readable } from "stream"; + +/** + * @public + * + * This union represents a superset of the compatible types you + * can use for streaming payload inputs. + * + * FAQ: + * Why does the type union mix mutually exclusive runtime types, namely + * Node.js and browser types? + * + * There are several reasons: + * 1. For backwards compatibility. + * 2. As a convenient compromise solution so that users in either environment may use the types + * without customization. + * 3. The SDK does not have static type information about the exact implementation + * of the HTTP RequestHandler being used in your client(s) (e.g. fetch, XHR, node:http, or node:http2), + * given that it is chosen at runtime. There are multiple possible request handlers + * in both the Node.js and browser runtime environments. + * + * Rather than restricting the type to a known common format (Uint8Array, for example) + * which doesn't include a universal streaming format in the currently supported Node.js versions, + * the type declaration is widened to multiple possible formats. + * It is up to the user to ultimately select a compatible format with the + * runtime and HTTP handler implementation they are using. + * + * Usage: + * The typical solution we expect users to have is to manually narrow the + * type when needed, picking the appropriate one out of the union according to the + * runtime environment and specific request handler. + * There is also the type utility "NodeJsClient", "BrowserClient" and more + * exported from this package. These can be applied at the client level + * to pre-narrow these streaming payload blobs. For usage see the readme.md + * in the root of the @smithy/types NPM package. + */ +export type StreamingBlobPayloadInputTypes = + | NodeJsRuntimeStreamingBlobPayloadInputTypes + | BrowserRuntimeStreamingBlobPayloadInputTypes; + +/** + * @public + * + * Streaming payload input types in the Node.js environment. + * These are derived from the types compatible with the request body used by node:http. + * + * Note: not all types are signable by the standard SignatureV4 signer when + * used as the request body. For example, in Node.js a Readable stream + * is not signable by the default signer. + * They are included in the union because it may be intended in some cases, + * but the expected types are primarily string, Uint8Array, and Buffer. + * + * Additional details may be found in the internal + * function "getPayloadHash" in the SignatureV4 module. + */ +export type NodeJsRuntimeStreamingBlobPayloadInputTypes = string | Uint8Array | Buffer | Readable; + +/** + * @public + * + * Streaming payload input types in the browser environment. + * These are derived from the types compatible with fetch's Request.body. + */ +export type BrowserRuntimeStreamingBlobPayloadInputTypes = string | Uint8Array | ReadableStream | Blob; diff --git a/packages/types/src/streaming-payload/streaming-blob-payload-output-types.ts b/packages/types/src/streaming-payload/streaming-blob-payload-output-types.ts new file mode 100644 index 00000000000..738a16e1319 --- /dev/null +++ b/packages/types/src/streaming-payload/streaming-blob-payload-output-types.ts @@ -0,0 +1,56 @@ +import type { IncomingMessage } from "http"; +import type { Readable } from "stream"; + +import type { SdkStream } from "../serde"; + +/** + * @public + * + * This union represents a superset of the types you may receive + * in streaming payload outputs. + * + * @see StreamingPayloadInputTypes for FAQ about mixing types from multiple environments. + * + * To highlight the upstream docs about the SdkStream mixin: + * + * The interface contains mix-in (via Object.assign) methods to transform the runtime-specific + * stream implementation to specified format. Each stream can ONLY be transformed + * once. + * + * The available methods are described on the SdkStream type via SdkStreamMixin. + */ +export type StreamingBlobPayloadOutputTypes = + | NodeJsRuntimeStreamingBlobPayloadOutputTypes + | BrowserRuntimeStreamingBlobPayloadOutputTypes; + +/** + * @public + * + * Streaming payload output types in the Node.js environment. + * + * This is by default the IncomingMessage type from node:http responses when + * using the default node-http-handler in Node.js environments. + * + * It can be other Readable types like node:http2's ClientHttp2Stream + * such as when using the node-http2-handler. + * + * The SdkStreamMixin adds methods on this type to help transform (collect) it to + * other formats. + */ +export type NodeJsRuntimeStreamingBlobPayloadOutputTypes = SdkStream; + +/** + * @public + * + * Streaming payload output types in the browser environment. + * + * This is by default fetch's Response.body type (ReadableStream) when using + * the default fetch-http-handler in browser-like environments. + * + * It may be a Blob, such as when using the XMLHttpRequest handler + * and receiving an arraybuffer response body. + * + * The SdkStreamMixin adds methods on this type to help transform (collect) it to + * other formats. + */ +export type BrowserRuntimeStreamingBlobPayloadOutputTypes = SdkStream; diff --git a/packages/types/src/transform/client-method-transforms.ts b/packages/types/src/transform/client-method-transforms.ts new file mode 100644 index 00000000000..e5f0dd147f6 --- /dev/null +++ b/packages/types/src/transform/client-method-transforms.ts @@ -0,0 +1,64 @@ +import type { Command } from "../command"; +import type { MetadataBearer } from "../response"; +import type { StreamingBlobPayloadOutputTypes } from "../streaming-payload/streaming-blob-payload-output-types"; +import type { Transform } from "./type-transform"; + +/** + * @internal + * + * Narrowed version of InvokeFunction used in Client::send. + */ +export interface NarrowedInvokeFunction< + NarrowType, + HttpHandlerOptions, + InputTypes extends object, + OutputTypes extends MetadataBearer, + ResolvedClientConfiguration +> { + ( + command: Command, + options?: HttpHandlerOptions + ): Promise>; + ( + command: Command, + cb: (err: unknown, data?: Transform) => void + ): void; + ( + command: Command, + options: HttpHandlerOptions, + cb: (err: unknown, data?: Transform) => void + ): void; + ( + command: Command, + options?: HttpHandlerOptions, + cb?: (err: unknown, data?: Transform) => void + ): Promise> | void; +} + +/** + * @internal + * + * Narrowed version of InvokeMethod used in aggregated Client methods. + */ +export interface NarrowedInvokeMethod< + NarrowType, + HttpHandlerOptions, + InputType extends object, + OutputType extends MetadataBearer +> { + (input: InputType, options?: HttpHandlerOptions): Promise< + Transform + >; + ( + input: InputType, + cb: (err: unknown, data?: Transform) => void + ): void; + ( + input: InputType, + options: HttpHandlerOptions, + cb: (err: unknown, data?: Transform) => void + ): void; + (input: InputType, options?: HttpHandlerOptions, cb?: (err: unknown, data?: OutputType) => void): Promise< + Transform + > | void; +} diff --git a/packages/types/src/transform/client-payload-blob-type-narrow.spec.ts b/packages/types/src/transform/client-payload-blob-type-narrow.spec.ts new file mode 100644 index 00000000000..f036035479a --- /dev/null +++ b/packages/types/src/transform/client-payload-blob-type-narrow.spec.ts @@ -0,0 +1,77 @@ +import type { IncomingMessage } from "node:http"; + +import type { Client } from "../client"; +import type { HttpHandlerOptions } from "../http"; +import type { MetadataBearer } from "../response"; +import type { SdkStream } from "../serde"; +import type { StreamingBlobPayloadOutputTypes } from "../streaming-payload/streaming-blob-payload-output-types"; +import type { BrowserClient, NodeJsClient } from "./client-payload-blob-type-narrow"; + +type Exact = [A] extends [B] ? ([B] extends [A] ? true : false) : false; + +// it should narrow operational methods and the generic send method + +type MyInput = Partial<{ + a: boolean; + b: boolean | number; + c: boolean | number | string; +}>; + +type MyOutput = { + a: boolean; + b: boolean | number; + c: boolean | number | string; + body?: StreamingBlobPayloadOutputTypes; +} & MetadataBearer; + +type MyConfig = { + version: number; +}; + +interface MyClient extends Client { + getObject(args: MyInput, options?: HttpHandlerOptions): Promise; + getObject(args: MyInput, cb: (err: any, data?: MyOutput) => void): void; + getObject(args: MyInput, options: HttpHandlerOptions, cb: (err: any, data?: MyOutput) => void): void; + + putObject(args: MyInput, options?: HttpHandlerOptions): Promise; + putObject(args: MyInput, cb: (err: any, data?: MyOutput) => void): void; + putObject(args: MyInput, options: HttpHandlerOptions, cb: (err: any, data?: MyOutput) => void): void; +} + +{ + interface NodeJsMyClient extends NodeJsClient {} + const mockClient = (null as unknown) as NodeJsMyClient; + const getObjectCall = () => mockClient.getObject({}); + + type A = Awaited>; + type B = Omit & { body?: SdkStream }; + + const assert1: Exact = true as const; +} + +{ + interface NodeJsMyClient extends BrowserClient {} + const mockClient = (null as unknown) as NodeJsMyClient; + const putObjectCall = () => + new Promise((resolve) => { + mockClient.putObject({}, (err: unknown, data) => { + resolve(data!); + }); + }); + + type A = Awaited>; + type B = Omit & { body?: SdkStream }; + + const assert1: Exact = true as const; +} + +{ + interface NodeJsMyClient extends NodeJsClient {} + const mockClient = (null as unknown) as NodeJsMyClient; + const sendCall = () => mockClient.send(null as any, { abortSignal: null as any }); + + type A = Awaited>; + type B = Omit & { body?: SdkStream }; + + const assert1: Exact = true as const; +} diff --git a/packages/types/src/transform/client-payload-blob-type-narrow.ts b/packages/types/src/transform/client-payload-blob-type-narrow.ts new file mode 100644 index 00000000000..e271ccbdb92 --- /dev/null +++ b/packages/types/src/transform/client-payload-blob-type-narrow.ts @@ -0,0 +1,77 @@ +import type { IncomingMessage } from "http"; +import type { ClientHttp2Stream } from "http2"; + +import type { InvokeFunction, InvokeMethod } from "../client"; +import type { HttpHandlerOptions } from "../http"; +import type { SdkStream } from "../serde"; +import type { NarrowedInvokeFunction, NarrowedInvokeMethod } from "./client-method-transforms"; + +/** + * @public + * + * Creates a type with a given client type that narrows payload blob output + * types to SdkStream. + * + * This can be used for clients with the NodeHttpHandler requestHandler, + * the default in Node.js when not using HTTP2. + * + * Usage example: + * ```typescript + * const client = new YourClient({}) as NodeJsClient; + * ``` + */ +export type NodeJsClient = NarrowPayloadBlobOutputType< + SdkStream, + ClientType +>; +/** + * @public + * Variant of NodeJsClient for node:http2. + */ +export type NodeJsHttp2Client = NarrowPayloadBlobOutputType< + SdkStream, + ClientType +>; + +/** + * @public + * + * Creates a type with a given client type that narrows payload blob output + * types to SdkStream. + * + * This can be used for clients with the FetchHttpHandler requestHandler, + * which is the default in browser environments. + * + * Usage example: + * ```typescript + * const client = new YourClient({}) as BrowserClient; + * ``` + */ +export type BrowserClient = NarrowPayloadBlobOutputType< + SdkStream, + ClientType +>; +/** + * @public + * + * Variant of BrowserClient for XMLHttpRequest. + */ +export type BrowserXhrClient = NarrowPayloadBlobOutputType< + SdkStream, + ClientType +>; + +/** + * @public + * + * Narrow a given Client's blob payload outputs to the given type T. + */ +export type NarrowPayloadBlobOutputType = { + [key in keyof ClientType]: [ClientType[key]] extends [ + InvokeFunction + ] + ? NarrowedInvokeFunction + : [ClientType[key]] extends [InvokeMethod] + ? NarrowedInvokeMethod + : ClientType[key]; +}; diff --git a/packages/types/src/transform/type-transform.spec.ts b/packages/types/src/transform/type-transform.spec.ts new file mode 100644 index 00000000000..bfec30c1322 --- /dev/null +++ b/packages/types/src/transform/type-transform.spec.ts @@ -0,0 +1,18 @@ +import type { Transform } from "./type-transform"; + +type Exact = [A] extends [B] ? ([B] extends [A] ? true : false) : false; + +// It should transform exact unions recursively. +type A = { + a: string; + b: number | string; + c: boolean | number | string; + nested: A; +}; + +type T = Transform; + +const assert1: Exact = true as const; +const assert2: Exact = true as const; + +const assert3: Exact = true as const; diff --git a/packages/types/src/transform/type-transform.ts b/packages/types/src/transform/type-transform.ts new file mode 100644 index 00000000000..a4d0522348b --- /dev/null +++ b/packages/types/src/transform/type-transform.ts @@ -0,0 +1,48 @@ +/** + * @public + * + * Transforms any members of the object T having type FromType + * to ToType. This applies only to exact type matches. + * + * This is for the case where FromType is a union and only those fields + * matching the same union should be transformed. + */ +export type Transform = ConditionalRecursiveTransformExact; + +/** + * @internal + * + * Returns ToType if T matches exactly with FromType. + */ +type TransformExact = [T] extends [FromType] ? ([FromType] extends [T] ? ToType : T) : T; + +/** + * @internal + * + * Applies TransformExact to members of an object recursively. + */ +type RecursiveTransformExact = T extends Function + ? T + : T extends object + ? { + [key in keyof T]: [T[key]] extends [FromType] + ? [FromType] extends [T[key]] + ? ToType + : ConditionalRecursiveTransformExact + : ConditionalRecursiveTransformExact; + } + : TransformExact; + +/** + * @internal + * + * Same as RecursiveTransformExact but does not assign to an object + * unless there is a matching transformed member. + */ +type ConditionalRecursiveTransformExact = [T] extends [ + RecursiveTransformExact +] + ? [RecursiveTransformExact] extends [T] + ? T + : RecursiveTransformExact + : RecursiveTransformExact;