feat(types): improve streaming payload types (#835)

.changeset/rare-dodos-exist.md

@@ -0,0 +1,5 @@
+---
+"@smithy/types": major
+---
+
+improved streaming payload types

.gitignore

@@ -36,6 +36,7 @@ smithy-typescript-integ-tests/yarn.lock
 
 # Issue https://github.com/awslabs/smithy-typescript/issues/425
 smithy-typescript-codegen/bin/
+smithy-typescript-ssdk-codegen-test-utils/bin/
 
 **/node_modules/
 **/*.tsbuildinfo

packages/types/README.md

@@ -2,3 +2,40 @@
 
 [![NPM version](https://img.shields.io/npm/v/@smithy/types/latest.svg)](https://www.npmjs.com/package/@smithy/types)
 [![NPM downloads](https://img.shields.io/npm/dm/@smithy/types.svg)](https://www.npmjs.com/package/@smithy/types)
+
+## Usage
+
+This package is mostly used internally by generated clients.
+Some public components have independent applications.
+
+### Scenario: Narrowing a smithy-typescript generated client's output payload blob types
+
+---
+
+This is mostly relevant to operations with streaming bodies such as within
+the S3Client in the AWS SDK for JavaScript v3.
+
+Because blob payload types are platform dependent, you may wish to indicate in your application that a client is running in a specific
+environment. This narrows the blob payload types.
+
+```typescript
+import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import type { NodeJsClient, SdkStream, StreamingBlobPayloadOutputTypes } from "@smithy/types";
+import type { IncomingMessage } from "node:http";
+
+// default client init.
+const s3Default = new S3Client({});
+
+// client init with type narrowing.
+const s3NarrowType = new S3Client({}) as NodeJsClient<S3Client>;
+
+// The default type of blob payloads is a wide union type including multiple possible
+// request handlers.
+const body1: StreamingBlobPayloadOutputTypes = (await s3Default.send(new GetObjectCommand({ Key: "", Bucket: "" })))
+  .Body!;
+
+// This is of the narrower type SdkStream<IncomingMessage> representing
+// blob payload responses using specifically the node:http request handler.
+const body2: SdkStream<IncomingMessage> = (await s3NarrowType.send(new GetObjectCommand({ Key: "", Bucket: "" })))
+  .Body!;
+```

packages/types/src/blob/blob-payload-input-types.ts

@@ -0,0 +1,48 @@
+import { Readable } from "stream";
+
+/**
+ * @public
+ *
+ * A union of types that can be used as inputs for the service model
+ * "blob" type when it represents the request's entire payload or body.
+ *
+ * For example, in Lambda::invoke, the payload is modeled as a blob type
+ * and this union applies to it.
+ * In contrast, in Lambda::createFunction the Zip file option is a blob type,
+ * but is not the (entire) payload and this union does not apply.
+ *
+ * Note: not all types are signable by the standard SignatureV4 signer when
+ * used as the request body. For example, in Node.js a Readable stream
+ * is not signable by the default signer.
+ * They are included in the union because it may work in some cases,
+ * but the expected types are primarily string and Uint8Array.
+ *
+ * Additional details may be found in the internal
+ * function "getPayloadHash" in the SignatureV4 module.
+ */
+export type BlobPayloadInputTypes =
+  | string
+  | ArrayBuffer
+  | ArrayBufferView
+  | Uint8Array
+  | NodeJsRuntimeBlobTypes
+  | BrowserRuntimeBlobTypes;
+
+/**
+ * @public
+ *
+ * Additional blob types for the Node.js environment.
+ */
+export type NodeJsRuntimeBlobTypes = Readable | Buffer;
+
+/**
+ * @public
+ *
+ * Additional blob types for the browser environment.
+ */
+export type BrowserRuntimeBlobTypes = Blob | ReadableStream;
+
+/**
+ * @deprecated renamed to BlobPayloadInputTypes.
+ */
+export type BlobTypes = BlobPayloadInputTypes;

packages/types/src/client.ts

@@ -7,11 +7,19 @@ import { MetadataBearer } from "./response";
  *
  * function definition for different overrides of client's 'send' function.
  */
-interface InvokeFunction<InputTypes extends object, OutputTypes extends MetadataBearer, ResolvedClientConfiguration> {
+export interface InvokeFunction<
+  InputTypes extends object,
+  OutputTypes extends MetadataBearer,
+  ResolvedClientConfiguration
+> {
   <InputType extends InputTypes, OutputType extends OutputTypes>(
     command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
     options?: any
   ): Promise<OutputType>;
+  <InputType extends InputTypes, OutputType extends OutputTypes>(
+    command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
+    cb: (err: any, data?: OutputType) => void
+  ): void;
   <InputType extends InputTypes, OutputType extends OutputTypes>(
     command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
     options: any,
@@ -24,6 +32,18 @@ interface InvokeFunction<InputTypes extends object, OutputTypes extends Metadata
   ): Promise<OutputType> | void;
 }
 
+/**
+ * @internal
+ *
+ * Signature that appears on aggregated clients' methods.
+ */
+export interface InvokeMethod<InputType extends object, OutputType extends MetadataBearer> {
+  (input: InputType, options?: any): Promise<OutputType>;
+  (input: InputType, cb: (err: any, data?: OutputType) => void): void;
+  (input: InputType, options: any, cb: (err: any, data?: OutputType) => void): void;
+  (input: InputType, options?: any, cb?: (err: any, data?: OutputType) => void): Promise<OutputType> | void;
+}
+
 /**
  * A general interface for service clients, idempotent to browser or node clients
  * This type corresponds to SmithyClient(https://github.com/aws/aws-sdk-js-v3/blob/main/packages/smithy-client/src/client.ts).

packages/types/src/index.ts

@@ -1,5 +1,6 @@
 export * from "./abort";
 export * from "./auth";
+export * from "./blob/blob-payload-input-types";
 export * from "./checksum";
 export * from "./client";
 export * from "./command";
@@ -21,7 +22,12 @@ export * from "./serde";
 export * from "./shapes";
 export * from "./signature";
 export * from "./stream";
+export * from "./streaming-payload/streaming-blob-common-types";
+export * from "./streaming-payload/streaming-blob-payload-input-types";
+export * from "./streaming-payload/streaming-blob-payload-output-types";
 export * from "./transfer";
+export * from "./transform/client-payload-blob-type-narrow";
+export * from "./transform/type-transform";
 export * from "./uri";
 export * from "./util";
 export * from "./waiter";

packages/types/src/streaming-payload/streaming-blob-common-types.ts

@@ -0,0 +1,34 @@
+import type { Readable } from "stream";
+
+/**
+ * @public
+ *
+ * This is the union representing the modeled blob type with streaming trait
+ * in a generic format that does not relate to HTTP input or output payloads.
+ *
+ * Note: the non-streaming blob type is represented by Uint8Array, but because
+ * the streaming blob type is always in the request/response paylod, it has
+ * historically been handled with different types.
+ *
+ * @see https://smithy.io/2.0/spec/simple-types.html#blob
+ *
+ * For compatibility with its historical representation, it must contain at least
+ * Readble (Node.js), Blob (browser), and ReadableStream (browser).
+ *
+ * @see StreamingPayloadInputTypes for FAQ about mixing types from multiple environments.
+ */
+export type StreamingBlobTypes = NodeJsRuntimeStreamingBlobTypes | BrowserRuntimeStreamingBlobTypes;
+
+/**
+ * @public
+ *
+ * Node.js streaming blob type.
+ */
+export type NodeJsRuntimeStreamingBlobTypes = Readable;
+
+/**
+ * @public
+ *
+ * Browser streaming blob types.
+ */
+export type BrowserRuntimeStreamingBlobTypes = ReadableStream | Blob;

packages/types/src/streaming-payload/streaming-blob-payload-input-types.ts

@@ -0,0 +1,64 @@
+import type { Readable } from "stream";
+
+/**
+ * @public
+ *
+ * This union represents a superset of the compatible types you
+ * can use for streaming payload inputs.
+ *
+ * FAQ:
+ * Why does the type union mix mutually exclusive runtime types, namely
+ * Node.js and browser types?
+ *
+ * There are several reasons:
+ * 1. For backwards compatibility.
+ * 2. As a convenient compromise solution so that users in either environment may use the types
+ *    without customization.
+ * 3. The SDK does not have static type information about the exact implementation
+ *    of the HTTP RequestHandler being used in your client(s) (e.g. fetch, XHR, node:http, or node:http2),
+ *    given that it is chosen at runtime. There are multiple possible request handlers
+ *    in both the Node.js and browser runtime environments.
+ *
+ *    Rather than restricting the type to a known common format (Uint8Array, for example)
+ *    which doesn't include a universal streaming format in the currently supported Node.js versions,
+ *    the type declaration is widened to multiple possible formats.
+ *    It is up to the user to ultimately select a compatible format with the
+ *    runtime and HTTP handler implementation they are using.
+ *
+ * Usage:
+ *    The typical solution we expect users to have is to manually narrow the
+ *    type when needed, picking the appropriate one out of the union according to the
+ *    runtime environment and specific request handler.
+ *    There is also the type utility "NodeJsClient", "BrowserClient" and more
+ *    exported from this package. These can be applied at the client level
+ *    to pre-narrow these streaming payload blobs. For usage see the readme.md
+ *    in the root of the @smithy/types NPM package.
+ */
+export type StreamingBlobPayloadInputTypes =
+  | NodeJsRuntimeStreamingBlobPayloadInputTypes
+  | BrowserRuntimeStreamingBlobPayloadInputTypes;
+
+/**
+ * @public
+ *
+ * Streaming payload input types in the Node.js environment.
+ * These are derived from the types compatible with the request body used by node:http.
+ *
+ * Note: not all types are signable by the standard SignatureV4 signer when
+ * used as the request body. For example, in Node.js a Readable stream
+ * is not signable by the default signer.
+ * They are included in the union because it may be intended in some cases,
+ * but the expected types are primarily string, Uint8Array, and Buffer.
+ *
+ * Additional details may be found in the internal
+ * function "getPayloadHash" in the SignatureV4 module.
+ */
+export type NodeJsRuntimeStreamingBlobPayloadInputTypes = string | Uint8Array | Buffer | Readable;
+
+/**
+ * @public
+ *
+ * Streaming payload input types in the browser environment.
+ * These are derived from the types compatible with fetch's Request.body.
+ */
+export type BrowserRuntimeStreamingBlobPayloadInputTypes = string | Uint8Array | ReadableStream | Blob;

packages/types/src/streaming-payload/streaming-blob-payload-output-types.ts

@@ -0,0 +1,56 @@
+import type { IncomingMessage } from "http";
+import type { Readable } from "stream";
+
+import type { SdkStream } from "../serde";
+
+/**
+ * @public
+ *
+ * This union represents a superset of the types you may receive
+ * in streaming payload outputs.
+ *
+ * @see StreamingPayloadInputTypes for FAQ about mixing types from multiple environments.
+ *
+ * To highlight the upstream docs about the SdkStream mixin:
+ *
+ * The interface contains mix-in (via Object.assign) methods to transform the runtime-specific
+ * stream implementation to specified format. Each stream can ONLY be transformed
+ * once.
+ *
+ * The available methods are described on the SdkStream type via SdkStreamMixin.
+ */
+export type StreamingBlobPayloadOutputTypes =
+  | NodeJsRuntimeStreamingBlobPayloadOutputTypes
+  | BrowserRuntimeStreamingBlobPayloadOutputTypes;
+
+/**
+ * @public
+ *
+ * Streaming payload output types in the Node.js environment.
+ *
+ * This is by default the IncomingMessage type from node:http responses when
+ * using the default node-http-handler in Node.js environments.
+ *
+ * It can be other Readable types like node:http2's ClientHttp2Stream
+ * such as when using the node-http2-handler.
+ *
+ * The SdkStreamMixin adds methods on this type to help transform (collect) it to
+ * other formats.
+ */
+export type NodeJsRuntimeStreamingBlobPayloadOutputTypes = SdkStream<IncomingMessage | Readable>;
+
+/**
+ * @public
+ *
+ * Streaming payload output types in the browser environment.
+ *
+ * This is by default fetch's Response.body type (ReadableStream) when using
+ * the default fetch-http-handler in browser-like environments.
+ *
+ * It may be a Blob, such as when using the XMLHttpRequest handler
+ * and receiving an arraybuffer response body.
+ *
+ * The SdkStreamMixin adds methods on this type to help transform (collect) it to
+ * other formats.
+ */
+export type BrowserRuntimeStreamingBlobPayloadOutputTypes = SdkStream<ReadableStream | Blob>;

packages/types/src/transform/client-method-transforms.ts

@@ -0,0 +1,64 @@
+import type { Command } from "../command";
+import type { MetadataBearer } from "../response";
+import type { StreamingBlobPayloadOutputTypes } from "../streaming-payload/streaming-blob-payload-output-types";
+import type { Transform } from "./type-transform";
+
+/**
+ * @internal
+ *
+ * Narrowed version of InvokeFunction used in Client::send.
+ */
+export interface NarrowedInvokeFunction<
+  NarrowType,
+  HttpHandlerOptions,
+  InputTypes extends object,
+  OutputTypes extends MetadataBearer,
+  ResolvedClientConfiguration
+> {
+  <InputType extends InputTypes, OutputType extends OutputTypes>(
+    command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
+    options?: HttpHandlerOptions
+  ): Promise<Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>>;
+  <InputType extends InputTypes, OutputType extends OutputTypes>(
+    command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
+    cb: (err: unknown, data?: Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>) => void
+  ): void;
+  <InputType extends InputTypes, OutputType extends OutputTypes>(
+    command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
+    options: HttpHandlerOptions,
+    cb: (err: unknown, data?: Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>) => void
+  ): void;
+  <InputType extends InputTypes, OutputType extends OutputTypes>(
+    command: Command<InputTypes, InputType, OutputTypes, OutputType, ResolvedClientConfiguration>,
+    options?: HttpHandlerOptions,
+    cb?: (err: unknown, data?: Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>) => void
+  ): Promise<Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>> | void;
+}
+
+/**
+ * @internal
+ *
+ * Narrowed version of InvokeMethod used in aggregated Client methods.
+ */
+export interface NarrowedInvokeMethod<
+  NarrowType,
+  HttpHandlerOptions,
+  InputType extends object,
+  OutputType extends MetadataBearer
+> {
+  (input: InputType, options?: HttpHandlerOptions): Promise<
+    Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>
+  >;
+  (
+    input: InputType,
+    cb: (err: unknown, data?: Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>) => void
+  ): void;
+  (
+    input: InputType,
+    options: HttpHandlerOptions,
+    cb: (err: unknown, data?: Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>) => void
+  ): void;
+  (input: InputType, options?: HttpHandlerOptions, cb?: (err: unknown, data?: OutputType) => void): Promise<
+    Transform<OutputType, StreamingBlobPayloadOutputTypes | undefined, NarrowType>
+  > | void;
+}