feat: encoders accept strings (#1176)

* feat: encoders accept strings

* add test cases

* typecheck in Encoder functions

Author: kuhe

.changeset/fluffy-deers-doubt.md

@@ -0,0 +1,8 @@
+---
+"@smithy/middleware-serde": minor
+"@smithy/util-base64": minor
+"@smithy/util-utf8": minor
+"@smithy/types": minor
+---
+
+encoders allow string inputs

packages/middleware-serde/src/deserializerMiddleware.spec.ts

@@ -1,3 +1,5 @@
+import { EndpointBearer, SerdeFunctions } from "@smithy/types";
+
 import { deserializerMiddleware } from "./deserializerMiddleware";
 
 describe("deserializerMiddleware", () => {
@@ -11,7 +13,7 @@ describe("deserializerMiddleware", () => {
         hostname: "hostname",
         path: "path",
       }),
-  };
+  } as EndpointBearer & SerdeFunctions;
 
   const mockArgs = {
     input: {

packages/middleware-serde/src/deserializerMiddleware.ts

@@ -5,11 +5,17 @@ import {
   DeserializeMiddleware,
   HandlerExecutionContext,
   ResponseDeserializer,
+  SerdeFunctions,
 } from "@smithy/types";
 
-export const deserializerMiddleware = <Input extends object, Output extends object, RuntimeUtils = any>(
-  options: RuntimeUtils,
-  deserializer: ResponseDeserializer<any, any, RuntimeUtils>
+/**
+ * @internal
+ *
+ * 3rd type parameter is deprecated and unused.
+ */
+export const deserializerMiddleware = <Input extends object, Output extends object, _ = any>(
+  options: SerdeFunctions,
+  deserializer: ResponseDeserializer<any, any, SerdeFunctions>
 ): DeserializeMiddleware<Input, Output> => (
   next: DeserializeHandler<Input, Output>,
   context: HandlerExecutionContext

packages/middleware-serde/src/serdePlugin.ts

@@ -8,6 +8,7 @@ import {
   Provider,
   RequestSerializer,
   ResponseDeserializer,
+  SerdeFunctions,
   SerializeHandlerOptions,
   UrlParser,
 } from "@smithy/types";
@@ -39,14 +40,19 @@ export type V1OrV2Endpoint = {
   endpoint?: Provider<Endpoint>;
 };
 
-export function getSerdePlugin<InputType extends object, SerDeContext, OutputType extends MetadataBearer>(
-  config: V1OrV2Endpoint,
-  serializer: RequestSerializer<any, SerDeContext & EndpointBearer>,
-  deserializer: ResponseDeserializer<OutputType, any, SerDeContext>
+/**
+ * @internal
+ *
+ * Note: 2nd type parameter is deprecated and unused.
+ */
+export function getSerdePlugin<InputType extends object, _, OutputType extends MetadataBearer>(
+  config: V1OrV2Endpoint & SerdeFunctions,
+  serializer: RequestSerializer<any, SerdeFunctions & EndpointBearer>,
+  deserializer: ResponseDeserializer<OutputType, any, SerdeFunctions>
 ): Pluggable<InputType, OutputType> {
   return {
     applyToStack: (commandStack: MiddlewareStack<InputType, OutputType>) => {
-      commandStack.add(deserializerMiddleware(config as SerDeContext, deserializer), deserializerMiddlewareOption);
+      commandStack.add(deserializerMiddleware(config, deserializer), deserializerMiddlewareOption);
       commandStack.add(serializerMiddleware(config, serializer), serializerMiddlewareOption);
     },
   };

packages/middleware-serde/src/serializerMiddleware.spec.ts

@@ -1,4 +1,4 @@
-import { EndpointBearer } from "@smithy/types";
+import { EndpointBearer, SerdeFunctions } from "@smithy/types";
 
 import { serializerMiddleware } from "./serializerMiddleware";
 
@@ -13,7 +13,7 @@ describe("serializerMiddleware", () => {
         hostname: "hostname",
         path: "path",
       }),
-  };
+  } as EndpointBearer & SerdeFunctions;
 
   const mockRequest = {
     method: "GET",

packages/middleware-serde/src/serializerMiddleware.ts

@@ -2,6 +2,7 @@ import {
   EndpointBearer,
   HandlerExecutionContext,
   RequestSerializer,
+  SerdeFunctions,
   SerializeHandler,
   SerializeHandlerArguments,
   SerializeHandlerOutput,
@@ -10,9 +11,14 @@ import {
 
 import type { V1OrV2Endpoint } from "./serdePlugin";
 
-export const serializerMiddleware = <Input extends object, Output extends object, RuntimeUtils extends EndpointBearer>(
-  options: V1OrV2Endpoint,
-  serializer: RequestSerializer<any, RuntimeUtils>
+/**
+ * @internal
+ *
+ * Note: 3rd type parameter is deprecated and unused.
+ */
+export const serializerMiddleware = <Input extends object, Output extends object, _>(
+  options: V1OrV2Endpoint & SerdeFunctions,
+  serializer: RequestSerializer<any, SerdeFunctions & EndpointBearer>
 ): SerializeMiddleware<Input, Output> => (
   next: SerializeHandler<Input, Output>,
   context: HandlerExecutionContext
@@ -28,7 +34,7 @@ export const serializerMiddleware = <Input extends object, Output extends object
     throw new Error("No valid endpoint provider available.");
   }
 
-  const request = await serializer(args.input, { ...options, endpoint } as RuntimeUtils);
+  const request = await serializer(args.input, { ...options, endpoint });
 
   return next({
     ...args,

packages/types/src/serde.ts

@@ -31,14 +31,22 @@ export interface StreamCollector {
  *
  * Request and Response serde util functions and settings for AWS services
  */
-export interface SerdeContext extends EndpointBearer {
+export interface SerdeContext extends SerdeFunctions, EndpointBearer {
+  requestHandler: RequestHandler<any, any>;
+  disableHostPrefix: boolean;
+}
+
+/**
+ * @public
+ *
+ * Serde functions from the client config.
+ */
+export interface SerdeFunctions {
   base64Encoder: Encoder;
   base64Decoder: Decoder;
   utf8Encoder: Encoder;
   utf8Decoder: Decoder;
   streamCollector: StreamCollector;
-  requestHandler: RequestHandler<any, any>;
-  disableHostPrefix: boolean;
 }
 
 /**

packages/types/src/util.ts

@@ -12,15 +12,23 @@ export type Exact<Type1, Type2> = [Type1] extends [Type2] ? ([Type2] extends [Ty
 /**
  * @public
  *
- * A function that, given a TypedArray of bytes, can produce a string
- * representation thereof.
+ * A function that, given a Uint8Array of bytes, can produce a string
+ * representation thereof. The function may optionally attempt to
+ * convert other input types to Uint8Array before encoding.
  *
  * @example An encoder function that converts bytes to hexadecimal
- * representation would return `'deadbeef'` when given
- * `new Uint8Array([0xde, 0xad, 0xbe, 0xef])`.
+ * representation would return `'hello'` when given
+ * `new Uint8Array([104, 101, 108, 108, 111])`.
  */
 export interface Encoder {
-  (input: Uint8Array): string;
+  /**
+   * Caution: the `any` type on the input is for backwards compatibility.
+   * Runtime support is limited to Uint8Array and string by default.
+   *
+   * You may choose to support more encoder input types if overriding the default
+   * implementations.
+   */
+  (input: Uint8Array | string | any): string;
 }
 
 /**
@@ -30,8 +38,8 @@ export interface Encoder {
  * string.
  *
  * @example A decoder function that converts bytes to hexadecimal
- * representation would return `new Uint8Array([0xde, 0xad, 0xbe, 0xef])` when
- * given the string `'deadbeef'`.
+ * representation would return `new Uint8Array([104, 101, 108, 108, 111])` when
+ * given the string `'hello'`.
  */
 export interface Decoder {
   (input: string): Uint8Array;

packages/util-base64/package.json

@@ -23,6 +23,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@smithy/util-buffer-from": "workspace:^",
+    "@smithy/util-utf8": "workspace:^",
     "tslib": "^2.5.0"
   },
   "devDependencies": {

packages/util-base64/src/toBase64.browser.spec.ts

@@ -1,11 +1,22 @@
 /**
  * @jest-environment jsdom
  */
+import type { Encoder } from "@smithy/types";
+
 import testCases from "./__mocks__/testCases.json";
 import { toBase64 } from "./toBase64.browser";
 
 describe(toBase64.name, () => {
   it.each(testCases as Array<[string, string, number[]]>)("%s", (desc, encoded, decoded) => {
     expect(toBase64(new Uint8Array(decoded))).toEqual(encoded);
   });
+
+  it("also converts strings", () => {
+    expect(toBase64("hello")).toEqual("aGVsbG8=");
+  });
+
+  it("throws on non-string non-Uint8Array", () => {
+    expect(() => (toBase64 as Encoder)(new Date())).toThrow();
+    expect(() => (toBase64 as Encoder)({})).toThrow();
+  });
 });

packages/util-base64/src/toBase64.browser.ts

@@ -1,12 +1,25 @@
+import { fromUtf8 } from "@smithy/util-utf8";
+
 import { alphabetByValue, bitsPerByte, bitsPerLetter, maxLetterValue } from "./constants.browser";
+
 /**
- * Converts a Uint8Array of binary data to a base-64 encoded string.
+ * Converts a Uint8Array of binary data or a utf-8 string to a base-64 encoded string.
  *
- * @param input The binary data to encode
+ * @param _input - the binary data or string to encode.
+ * @returns base64 string.
  *
  * @see https://tools.ietf.org/html/rfc4648#section-4
  */
-export function toBase64(input: Uint8Array): string {
+export function toBase64(_input: Uint8Array | string): string {
+  let input: Uint8Array;
+  if (typeof _input === "string") {
+    input = fromUtf8(_input);
+  } else {
+    input = _input as Uint8Array;
+  }
+  if (typeof input !== "object" || typeof input.byteOffset !== "number" || typeof input.byteLength !== "number") {
+    throw new Error("@smithy/util-base64: toBase64 encoder function only accepts string | Uint8Array.");
+  }
   let str = "";
   for (let i = 0; i < input.length; i += 3) {
     let bits = 0;

packages/util-base64/src/toBase64.spec.ts

@@ -1,3 +1,5 @@
+import type { Encoder } from "@smithy/types";
+
 import testCases from "./__mocks__/testCases.json";
 import { toBase64 } from "./toBase64";
 
@@ -9,4 +11,13 @@ describe(toBase64.name, () => {
   it("should throw when given a number", () => {
     expect(() => toBase64(0xdeadbeefface as any)).toThrow();
   });
+
+  it("also converts strings", () => {
+    expect(toBase64("hello")).toEqual("aGVsbG8=");
+  });
+
+  it("throws on non-string non-Uint8Array", () => {
+    expect(() => (toBase64 as Encoder)(new Date())).toThrow();
+    expect(() => (toBase64 as Encoder)({})).toThrow();
+  });
 });

packages/util-base64/src/toBase64.ts

@@ -1,10 +1,22 @@
 import { fromArrayBuffer } from "@smithy/util-buffer-from";
+import { fromUtf8 } from "@smithy/util-utf8";
 
 /**
- * Converts a Uint8Array of binary data to a base-64 encoded string using
+ * Converts a Uint8Array of binary data or a utf-8 string to a base-64 encoded string using
  * Node.JS's `buffer` module.
  *
- * @param input The binary data to encode
+ * @param _input - the binary data or string to encode.
+ * @returns base64 string.
  */
-export const toBase64 = (input: Uint8Array): string =>
-  fromArrayBuffer(input.buffer, input.byteOffset, input.byteLength).toString("base64");
+export const toBase64 = (_input: Uint8Array | string): string => {
+  let input: Uint8Array;
+  if (typeof _input === "string") {
+    input = fromUtf8(_input);
+  } else {
+    input = _input as Uint8Array;
+  }
+  if (typeof input !== "object" || typeof input.byteOffset !== "number" || typeof input.byteLength !== "number") {
+    throw new Error("@smithy/util-base64: toBase64 encoder function only accepts string | Uint8Array.");
+  }
+  return fromArrayBuffer(input.buffer, input.byteOffset, input.byteLength).toString("base64");
+};

packages/util-utf8/src/toUtf8.browser.spec.ts

@@ -1,6 +1,8 @@
 /**
  * @jest-environment jsdom
  */
+import type { Encoder } from "@smithy/types";
+
 import { toUtf8 } from "./toUtf8.browser";
 
 declare const global: any;
@@ -13,4 +15,13 @@ describe("toUtf8", () => {
 
     expect(toUtf8(new Uint8Array(0))).toBe(expected);
   });
+
+  it("passes through strings", () => {
+    expect(toUtf8("hello")).toEqual("hello");
+  });
+
+  it("throws on non-string non-Uint8Array", () => {
+    expect(() => (toUtf8 as Encoder)(new Date())).toThrow();
+    expect(() => (toUtf8 as Encoder)({})).toThrow();
+  });
 });

packages/util-utf8/src/toUtf8.browser.ts

@@ -1 +1,15 @@
-export const toUtf8 = (input: Uint8Array): string => new TextDecoder("utf-8").decode(input);
+/**
+ *
+ * This does not convert non-utf8 strings to utf8, it only passes through strings if
+ * a string is received instead of a Uint8Array.
+ *
+ */
+export const toUtf8 = (input: Uint8Array | string): string => {
+  if (typeof input === "string") {
+    return input;
+  }
+  if (typeof input !== "object" || typeof input.byteOffset !== "number" || typeof input.byteLength !== "number") {
+    throw new Error("@smithy/util-utf8: toUtf8 encoder function only accepts string | Uint8Array.");
+  }
+  return new TextDecoder("utf-8").decode(input);
+};

packages/util-utf8/src/toUtf8.spec.ts

@@ -1,3 +1,5 @@
+import type { Encoder } from "@smithy/types";
+
 import { toUtf8 } from "./toUtf8";
 
 const utf8StringsToByteArrays: Record<string, Uint8Array> = {
@@ -144,4 +146,13 @@ describe("toUtf8", () => {
   it("should throw when given a number", () => {
     expect(() => toUtf8(255 as any)).toThrow();
   });
+
+  it("passes through strings", () => {
+    expect(toUtf8("hello")).toEqual("hello");
+  });
+
+  it("throws on non-string non-Uint8Array", () => {
+    expect(() => (toUtf8 as Encoder)(new Date())).toThrow();
+    expect(() => (toUtf8 as Encoder)({})).toThrow();
+  });
 });

packages/util-utf8/src/toUtf8.ts

@@ -1,4 +1,17 @@
 import { fromArrayBuffer } from "@smithy/util-buffer-from";
 
-export const toUtf8 = (input: Uint8Array): string =>
-  fromArrayBuffer(input.buffer, input.byteOffset, input.byteLength).toString("utf8");
+/**
+ *
+ * This does not convert non-utf8 strings to utf8, it only passes through strings if
+ * a string is received instead of a Uint8Array.
+ *
+ */
+export const toUtf8 = (input: Uint8Array | string): string => {
+  if (typeof input === "string") {
+    return input;
+  }
+  if (typeof input !== "object" || typeof input.byteOffset !== "number" || typeof input.byteLength !== "number") {
+    throw new Error("@smithy/util-utf8: toUtf8 encoder function only accepts string | Uint8Array.");
+  }
+  return fromArrayBuffer(input.buffer, input.byteOffset, input.byteLength).toString("utf8");
+};

yarn.lock

@@ -2554,6 +2554,7 @@ __metadata:
   resolution: "@smithy/util-base64@workspace:packages/util-base64"
   dependencies:
     "@smithy/util-buffer-from": "workspace:^"
+    "@smithy/util-utf8": "workspace:^"
     "@tsconfig/recommended": 1.0.1
     "@types/node": ^14.14.31
     concurrently: 7.0.0