Skip to content

Commit f7c9674

Browse files
stainless-app[bot]stainless-app[bot]
committed
chore(internal): move LineDecoder to a separate file (#1120)
1 parent 785ef4b commit f7c9674

File tree

2 files changed

+115
-111
lines changed

2 files changed

+115
-111
lines changed

‎src/internal/decoders/line.ts

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { OpenAIError } from '../../error';
2+
3+
type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;
4+
5+
/**
6+
* A re-implementation of httpx's `LineDecoder` in Python that handles incrementally
7+
* reading lines from text.
8+
*
9+
* https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258
10+
*/
11+
export class LineDecoder {
12+
// prettier-ignore
13+
static NEWLINE_CHARS = new Set(['\n', '\r']);
14+
static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
15+
16+
buffer: string[];
17+
trailingCR: boolean;
18+
textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
19+
20+
constructor() {
21+
this.buffer = [];
22+
this.trailingCR = false;
23+
}
24+
25+
decode(chunk: Bytes): string[] {
26+
let text = this.decodeText(chunk);
27+
28+
if (this.trailingCR) {
29+
text = '\r' + text;
30+
this.trailingCR = false;
31+
}
32+
if (text.endsWith('\r')) {
33+
this.trailingCR = true;
34+
text = text.slice(0, -1);
35+
}
36+
37+
if (!text) {
38+
return [];
39+
}
40+
41+
const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
42+
let lines = text.split(LineDecoder.NEWLINE_REGEXP);
43+
44+
// if there is a trailing new line then the last entry will be an empty
45+
// string which we don't care about
46+
if (trailingNewline) {
47+
lines.pop();
48+
}
49+
50+
if (lines.length === 1 && !trailingNewline) {
51+
this.buffer.push(lines[0]!);
52+
return [];
53+
}
54+
55+
if (this.buffer.length > 0) {
56+
lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
57+
this.buffer = [];
58+
}
59+
60+
if (!trailingNewline) {
61+
this.buffer = [lines.pop() || ''];
62+
}
63+
64+
return lines;
65+
}
66+
67+
decodeText(bytes: Bytes): string {
68+
if (bytes == null) return '';
69+
if (typeof bytes === 'string') return bytes;
70+
71+
// Node:
72+
if (typeof Buffer !== 'undefined') {
73+
if (bytes instanceof Buffer) {
74+
return bytes.toString();
75+
}
76+
if (bytes instanceof Uint8Array) {
77+
return Buffer.from(bytes).toString();
78+
}
79+
80+
throw new OpenAIError(
81+
`Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`,
82+
);
83+
}
84+
85+
// Browser
86+
if (typeof TextDecoder !== 'undefined') {
87+
if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {
88+
this.textDecoder ??= new TextDecoder('utf8');
89+
return this.textDecoder.decode(bytes);
90+
}
91+
92+
throw new OpenAIError(
93+
`Unexpected: received non-Uint8Array/ArrayBuffer (${
94+
(bytes as any).constructor.name
95+
}) in a web platform. Please report this error.`,
96+
);
97+
}
98+
99+
throw new OpenAIError(
100+
`Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,
101+
);
102+
}
103+
104+
flush(): string[] {
105+
if (!this.buffer.length && !this.trailingCR) {
106+
return [];
107+
}
108+
109+
const lines = [this.buffer.join('')];
110+
this.buffer = [];
111+
this.trailingCR = false;
112+
return lines;
113+
}
114+
}

‎src/streaming.ts

Lines changed: 1 addition & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { ReadableStream, type Response } from './_shims/index';
22
import { OpenAIError } from './error';
3+
import { LineDecoder } from './internal/decoders/line';
34

45
import { APIError } from 'openai/error';
56

@@ -343,117 +344,6 @@ class SSEDecoder {
343344
}
344345
}
345346

346-
/**
347-
* A re-implementation of httpx's `LineDecoder` in Python that handles incrementally
348-
* reading lines from text.
349-
*
350-
* https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258
351-
*/
352-
class LineDecoder {
353-
// prettier-ignore
354-
static NEWLINE_CHARS = new Set(['\n', '\r']);
355-
static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
356-
357-
buffer: string[];
358-
trailingCR: boolean;
359-
textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
360-
361-
constructor() {
362-
this.buffer = [];
363-
this.trailingCR = false;
364-
}
365-
366-
decode(chunk: Bytes): string[] {
367-
let text = this.decodeText(chunk);
368-
369-
if (this.trailingCR) {
370-
text = '\r' + text;
371-
this.trailingCR = false;
372-
}
373-
if (text.endsWith('\r')) {
374-
this.trailingCR = true;
375-
text = text.slice(0, -1);
376-
}
377-
378-
if (!text) {
379-
return [];
380-
}
381-
382-
const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
383-
let lines = text.split(LineDecoder.NEWLINE_REGEXP);
384-
385-
// if there is a trailing new line then the last entry will be an empty
386-
// string which we don't care about
387-
if (trailingNewline) {
388-
lines.pop();
389-
}
390-
391-
if (lines.length === 1 && !trailingNewline) {
392-
this.buffer.push(lines[0]!);
393-
return [];
394-
}
395-
396-
if (this.buffer.length > 0) {
397-
lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
398-
this.buffer = [];
399-
}
400-
401-
if (!trailingNewline) {
402-
this.buffer = [lines.pop() || ''];
403-
}
404-
405-
return lines;
406-
}
407-
408-
decodeText(bytes: Bytes): string {
409-
if (bytes == null) return '';
410-
if (typeof bytes === 'string') return bytes;
411-
412-
// Node:
413-
if (typeof Buffer !== 'undefined') {
414-
if (bytes instanceof Buffer) {
415-
return bytes.toString();
416-
}
417-
if (bytes instanceof Uint8Array) {
418-
return Buffer.from(bytes).toString();
419-
}
420-
421-
throw new OpenAIError(
422-
`Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`,
423-
);
424-
}
425-
426-
// Browser
427-
if (typeof TextDecoder !== 'undefined') {
428-
if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {
429-
this.textDecoder ??= new TextDecoder('utf8');
430-
return this.textDecoder.decode(bytes);
431-
}
432-
433-
throw new OpenAIError(
434-
`Unexpected: received non-Uint8Array/ArrayBuffer (${
435-
(bytes as any).constructor.name
436-
}) in a web platform. Please report this error.`,
437-
);
438-
}
439-
440-
throw new OpenAIError(
441-
`Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,
442-
);
443-
}
444-
445-
flush(): string[] {
446-
if (!this.buffer.length && !this.trailingCR) {
447-
return [];
448-
}
449-
450-
const lines = [this.buffer.join('')];
451-
this.buffer = [];
452-
this.trailingCR = false;
453-
return lines;
454-
}
455-
}
456-
457347
/** This is an internal helper function that's just used for testing */
458348
export function _decodeChunks(chunks: string[]): string[] {
459349
const decoder = new LineDecoder();

0 commit comments

Comments
 (0)