Override serialisation in node

Author: timfish

packages/browser/jest.config.js

@@ -2,6 +2,6 @@ const baseConfig = require('../../jest/jest.config.js');
 
 module.exports = {
   ...baseConfig,
-  testEnvironment: 'jsdom',
+  testEnvironment: './jest.env.js',
   testMatch: ['<rootDir>/test/unit/**/*.test.ts'],
 };

packages/browser/jest.env.js

@@ -0,0 +1,15 @@
+const Environment = require('jest-environment-jsdom');
+
+// Looks like jsdom does not support global TextEncoder/TextDecoder
+// https://github.com/jsdom/jsdom/issues/2524
+
+module.exports = class CustomTestEnvironment extends Environment {
+  async setup() {
+    await super.setup();
+    if (typeof this.global.TextEncoder === 'undefined') {
+      const { TextEncoder, TextDecoder } = require('util');
+      this.global.TextEncoder = TextEncoder;
+      this.global.TextDecoder = TextDecoder;
+    }
+  }
+};

packages/core/src/transports/base.ts

@@ -69,7 +69,7 @@ export function createTransport(
     };
 
     const requestTask = (): PromiseLike<void> =>
-      makeRequest({ body: serializeEnvelope(filteredEnvelope) }).then(
+      makeRequest({ body: (options.serializeEnvelope || serializeEnvelope)(filteredEnvelope) }).then(
         ({ headers }): void => {
           if (headers) {
             rateLimits = updateRateLimits(rateLimits, headers);

packages/hub/src/scope.ts

@@ -1,7 +1,6 @@
 /* eslint-disable max-lines */
 import {
   Attachment,
-  AttachmentOptions,
   Breadcrumb,
   CaptureContext,
   Context,
@@ -408,8 +407,8 @@ export class Scope implements ScopeInterface {
   /**
    * @inheritDoc
    */
-  public addAttachment(pathOrData: string | Uint8Array, options?: AttachmentOptions): this {
-    this._attachments.push([pathOrData, options]);
+  public addAttachment(attachment: Attachment): this {
+    this._attachments.push(attachment);
     return this;
   }
 

packages/node/src/client.ts

@@ -158,15 +158,12 @@ export class NodeClient extends BaseClient<NodeClientOptions> {
   protected _attachmentsFromScope(scope: Scope | undefined): AttachmentItem[] {
     return (
       scope?.getAttachments()?.map(attachment => {
-        let [pathOrData, options] = attachment;
-
-        if (typeof pathOrData === 'string' && existsSync(pathOrData)) {
-          options = options || {};
-          options.filename = basename(pathOrData);
-          pathOrData = readFileSync(pathOrData);
+        if (attachment.path && existsSync(attachment.path)) {
+          attachment.filename = basename(attachment.path);
+          attachment.data = readFileSync(attachment.path);
         }
 
-        return createAttachmentEnvelopeItem([pathOrData, options]);
+        return createAttachmentEnvelopeItem(attachment);
       }) || []
     );
   }

packages/node/src/transports/http.ts

@@ -6,6 +6,7 @@ import {
   TransportRequest,
   TransportRequestExecutor,
 } from '@sentry/types';
+import { serializeEnvelope } from '@sentry/utils';
 import * as http from 'http';
 import * as https from 'https';
 import { Readable, Writable } from 'stream';
@@ -45,6 +46,8 @@ function streamFromBody(body: Uint8Array | string): Readable {
  * Creates a Transport that uses native the native 'http' and 'https' modules to send events to Sentry.
  */
 export function makeNodeTransport(options: NodeTransportOptions): Transport {
+  options.serializeEnvelope = s => serializeEnvelope(s, new TextEncoder());
+
   const urlSegments = new URL(options.url);
   const isHttps = urlSegments.protocol === 'https:';
 

packages/node/test/transports/http.test.ts

@@ -2,6 +2,7 @@ import { createTransport } from '@sentry/core';
 import { EventEnvelope, EventItem } from '@sentry/types';
 import { createEnvelope, serializeEnvelope } from '@sentry/utils';
 import * as http from 'http';
+import { TextEncoder, TextDecoder } from 'util';
 
 import { makeNodeTransport } from '../../src/transports';
 
@@ -66,7 +67,7 @@ const EVENT_ENVELOPE = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4b
   [{ type: 'event' }, { event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2' }] as EventItem,
 ]);
 
-const SERIALIZED_EVENT_ENVELOPE = serializeEnvelope(EVENT_ENVELOPE);
+const SERIALIZED_EVENT_ENVELOPE = new TextDecoder().decode(serializeEnvelope(EVENT_ENVELOPE, new TextEncoder()));
 
 const defaultOptions = {
   url: TEST_SERVER_URL,

packages/node/test/transports/https.test.ts

@@ -3,6 +3,7 @@ import { EventEnvelope, EventItem } from '@sentry/types';
 import { createEnvelope, serializeEnvelope } from '@sentry/utils';
 import * as http from 'http';
 import * as https from 'https';
+import { TextEncoder } from 'util';
 
 import { makeNodeTransport } from '../../src/transports';
 import { HTTPModule, HTTPModuleRequestIncomingMessage } from '../../src/transports/http-module';
@@ -69,7 +70,7 @@ const EVENT_ENVELOPE = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4b
   [{ type: 'event' }, { event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2' }] as EventItem,
 ]);
 
-const SERIALIZED_EVENT_ENVELOPE = serializeEnvelope(EVENT_ENVELOPE);
+const SERIALIZED_EVENT_ENVELOPE = new TextDecoder().decode(serializeEnvelope(EVENT_ENVELOPE, new TextEncoder()));
 
 const unsafeHttpsModule: HTTPModule = {
   request: jest

packages/types/src/attachment.ts

@@ -1,7 +1,7 @@
-export interface AttachmentOptions {
+export interface Attachment {
+  path?: string;
+  data?: string | Uint8Array;
   filename?: string;
   contentType?: string;
   attachmentType?: string;
 }
-
-export type Attachment = [string | Uint8Array, AttachmentOptions | undefined];

packages/types/src/index.ts

@@ -1,4 +1,4 @@
-export type { Attachment, AttachmentOptions } from './attachment';
+export type { Attachment } from './attachment';
 export type { Breadcrumb, BreadcrumbHint } from './breadcrumb';
 export type { Client } from './client';
 export type { ClientReport, Outcome, EventDropReason } from './clientreport';

packages/types/src/scope.ts

@@ -1,4 +1,4 @@
-import { Attachment, AttachmentOptions } from './attachment';
+import { Attachment } from './attachment';
 import { Breadcrumb } from './breadcrumb';
 import { Context, Contexts } from './context';
 import { EventProcessor } from './eventprocessor';
@@ -162,10 +162,9 @@ export interface Scope {
 
   /**
    * Adds an attachment to the scope
-   * @param pathOrData A Uint8Array containing the attachment bytes
-   * @param options Attachment options
+   * @param attachment Attachment options
    */
-  addAttachment(pathOrData: string | Uint8Array, options?: AttachmentOptions): this;
+  addAttachment(attachment: Attachment): this;
 
   /**
    * Returns an array of attachments on the scope

packages/types/src/transport.ts

@@ -17,6 +17,7 @@ export type TransportMakeRequestResponse = {
 export interface InternalBaseTransportOptions {
   bufferSize?: number;
   recordDroppedEvent: (reason: EventDropReason, dataCategory: DataCategory) => void;
+  serializeEnvelope?: (env: Envelope) => Uint8Array;
 }
 
 export interface BaseTransportOptions extends InternalBaseTransportOptions {

packages/utils/src/envelope.ts

@@ -1,7 +1,5 @@
 import { Attachment, AttachmentItem, DataCategory, Envelope, EnvelopeItem, EnvelopeItemType } from '@sentry/types';
 
-import { isPrimitive } from './is';
-
 /**
  * Creates an envelope.
  * Make sure to always explicitly provide the generic to this function
@@ -36,61 +34,48 @@ export function forEachEnvelopeItem<E extends Envelope>(
   });
 }
 
-/**
- * Serializes an envelope.
- */
-export function serializeEnvelope(envelope: Envelope): string | Uint8Array {
-  const [, items] = envelope;
-
-  // Have to cast items to any here since Envelope is a union type
-  // Fixed in Typescript 4.2
-  // TODO: Remove any[] cast when we upgrade to TS 4.2
-  // https://github.com/microsoft/TypeScript/issues/36390
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const hasBinaryAttachment = (items as any[]).some(
-    (item: typeof items[number]) => item[0].type === 'attachment' && item[1] instanceof Uint8Array,
-  );
-
-  return hasBinaryAttachment ? serializeBinaryEnvelope(envelope) : serializeStringEnvelope(envelope);
+// Cached string encoder
+let encoder: TextEncoder | undefined;
+
+function getCachedEncoder(): TextEncoder {
+  if (!encoder) {
+    encoder = new TextEncoder();
+  }
+
+  return encoder;
+}
+
+// Global TextEncoder and Node require('util').TextEncoder
+interface TextEncoderInternal extends TextEncoderCommon {
+  encode(input?: string): Uint8Array;
 }
 
 /**
- * Serializes an envelope to a string.
+ * Serializes an envelope.
  */
-export function serializeStringEnvelope(envelope: Envelope): string {
-  const [headers, items] = envelope;
-  const serializedHeaders = JSON.stringify(headers);
-
-  // TODO: Remove any[] cast when we upgrade to TS 4.2
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  return (items as any[]).reduce((acc, item: typeof items[number]) => {
-    const [itemHeaders, payload] = item;
-    // We do not serialize payloads that are primitives
-    const serializedPayload = isPrimitive(payload) ? String(payload) : JSON.stringify(payload);
-    return `${acc}\n${JSON.stringify(itemHeaders)}\n${serializedPayload}`;
-  }, serializedHeaders);
-}
+export function serializeEnvelope(envelope: Envelope, textEncoderOverride?: TextEncoderInternal): Uint8Array {
+  const textEncoder = textEncoderOverride || getCachedEncoder();
 
-function serializeBinaryEnvelope(envelope: Envelope): Uint8Array {
-  const encoder = new TextEncoder();
   const [headers, items] = envelope;
-  const serializedHeaders = JSON.stringify(headers);
 
-  const chunks = [encoder.encode(serializedHeaders)];
+  const parts = [textEncoder.encode(JSON.stringify(headers))];
 
   for (const item of items) {
     const [itemHeaders, payload] = item as typeof items[number];
-    chunks.push(encoder.encode(`\n${JSON.stringify(itemHeaders)}\n`));
+    parts.push(textEncoder.encode(`\n${JSON.stringify(itemHeaders)}\n`));
+
     if (typeof payload === 'string') {
-      chunks.push(encoder.encode(payload));
+      parts.push(textEncoder.encode(payload));
     } else if (payload instanceof Uint8Array) {
-      chunks.push(payload);
+      parts.push(payload);
     } else {
-      chunks.push(encoder.encode(JSON.stringify(payload)));
+      parts.push(textEncoder.encode(JSON.stringify(payload)));
     }
   }
 
-  return concatBuffers(chunks);
+  parts.push(textEncoder.encode('\n'));
+
+  return concatBuffers(parts);
 }
 
 function concatBuffers(buffers: Uint8Array[]): Uint8Array {
@@ -110,19 +95,17 @@ function concatBuffers(buffers: Uint8Array[]): Uint8Array {
  * Creates attachment envelope items
  */
 export function createAttachmentEnvelopeItem(attachment: Attachment): AttachmentItem {
-  const [pathOrData, options] = attachment;
-
-  const buffer = typeof pathOrData === 'string' ? new TextEncoder().encode(pathOrData) : pathOrData;
+  const buffer = typeof attachment.data === 'string' ? new TextEncoder().encode(attachment.data) : attachment.data;
 
   return [
     {
       type: 'attachment',
-      length: buffer.length,
-      filename: options?.filename || 'No filename',
-      content_type: options?.contentType,
-      attachment_type: options?.attachmentType,
+      length: buffer?.length || 0,
+      filename: attachment?.filename || 'No filename',
+      content_type: attachment?.contentType,
+      attachment_type: attachment?.attachmentType,
     },
-    buffer,
+    buffer || new Uint8Array(0),
   ];
 }
 

packages/utils/test/clientreport.test.ts

@@ -42,10 +42,12 @@ describe('createClientReportEnvelope', () => {
   it('serializes an envelope', () => {
     const env = createClientReportEnvelope(DEFAULT_DISCARDED_EVENTS, MOCK_DSN, 123456);
     const serializedEnv = serializeEnvelope(env);
-    expect(serializedEnv).toMatchInlineSnapshot(`
-      "{\\"dsn\\":\\"https://public@example.com/1\\"}
-      {\\"type\\":\\"client_report\\"}
-      {\\"timestamp\\":123456,\\"discarded_events\\":[{\\"reason\\":\\"before_send\\",\\"category\\":\\"error\\",\\"quantity\\":30},{\\"reason\\":\\"network_error\\",\\"category\\":\\"transaction\\",\\"quantity\\":23}]}"
-    `);
+
+    const strEnvelope = new TextDecoder().decode(serializedEnv);
+
+    expect(strEnvelope).toMatch(`{"dsn":"https://public@example.com/1"}
+{"type":"client_report"}
+{"timestamp":123456,"discarded_events":[{"reason":"before_send","category":"error","quantity":30},{"reason":"network_error","category":"transaction","quantity":23}]}
+`);
   });
 });

packages/utils/test/envelope.test.ts

@@ -1,12 +1,6 @@
 import { EventEnvelope } from '@sentry/types';
 
-import {
-  addItemToEnvelope,
-  createEnvelope,
-  forEachEnvelopeItem,
-  serializeStringEnvelope,
-  serializeEnvelope,
-} from '../src/envelope';
+import { addItemToEnvelope, createEnvelope, forEachEnvelopeItem, serializeEnvelope } from '../src/envelope';
 import { parseEnvelope } from './testutils';
 
 describe('envelope', () => {
@@ -23,12 +17,11 @@ describe('envelope', () => {
     });
   });
 
-  describe('serializeStringEnvelope()', () => {
+  describe('serializeEnvelope()', () => {
     it('serializes an envelope', () => {
       const env = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2', sent_at: '123' }, []);
-      expect(serializeStringEnvelope(env)).toMatchInlineSnapshot(
-        '"{\\"event_id\\":\\"aa3ff046696b4bc6b609ce6d28fde9e2\\",\\"sent_at\\":\\"123\\"}"',
-      );
+      const strEnvelope = new TextDecoder().decode(serializeEnvelope(env));
+      expect(strEnvelope).toMatch('{"event_id":"aa3ff046696b4bc6b609ce6d28fde9e2","sent_at":"123"}');
     });
   });
 

packages/utils/test/testutils.ts

@@ -1,4 +1,4 @@
-import { BaseEnvelopeHeaders, BaseEnvelopeItemHeaders,Envelope } from '@sentry/types';
+import { BaseEnvelopeHeaders, BaseEnvelopeItemHeaders, Envelope } from '@sentry/types';
 
 export const testOnlyIfNodeVersionAtLeast = (minVersion: number): jest.It => {
   const currentNodeVersion = process.env.NODE_VERSION;
@@ -18,32 +18,30 @@ export const testOnlyIfNodeVersionAtLeast = (minVersion: number): jest.It => {
  * A naive binary envelope parser
  */
 export function parseEnvelope(env: string | Uint8Array): Envelope {
-  if (typeof env === 'string') {
-    env = new TextEncoder().encode(env);
-  }
+  let buf = typeof env === 'string' ? new TextEncoder().encode(env) : env;
 
   let envelopeHeaders: BaseEnvelopeHeaders | undefined;
   let lastItemHeader: BaseEnvelopeItemHeaders | undefined;
   const items: [any, any][] = [];
 
   let binaryLength = 0;
-  while (env.length) {
+  while (buf.length) {
     // Next length is either the binary length from the previous header
     // or the next newline character
-    let i = binaryLength || env.indexOf(0xa);
+    let i = binaryLength || buf.indexOf(0xa);
 
     // If no newline was found, assume this is the last block
     if (i < 0) {
-      i = env.length;
+      i = buf.length;
     }
 
     // If we read out a length in the previous header, assume binary
     if (binaryLength > 0) {
-      const bin = env.slice(0, binaryLength);
+      const bin = buf.slice(0, binaryLength);
       binaryLength = 0;
       items.push([lastItemHeader, bin]);
     } else {
-      const json = JSON.parse(new TextDecoder().decode(env.slice(0, i + 1)));
+      const json = JSON.parse(new TextDecoder().decode(buf.slice(0, i + 1)));
 
       if (typeof json.length === 'number') {
         binaryLength = json.length;
@@ -63,7 +61,7 @@ export function parseEnvelope(env: string | Uint8Array): Envelope {
     }
 
     // Replace the buffer with the previous block and newline removed
-    env = env.slice(i + 1);
+    buf = buf.slice(i + 1);
   }
 
   return [envelopeHeaders as BaseEnvelopeHeaders, items];