diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 36f50eb..ea09a23 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,9 +12,8 @@ ------------------------------------------------------------------------------------------------- - + - [ ] I prefixed the PR-title with `docs: `, `fix(area): `, `feat(area): ` or `breaking(area): ` -- [ ] I updated ./CHANGELOG.md with a link to this PR or Issue - [ ] I updated the README.md - [ ] I Added unit test(s) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9fad5..d352efc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,8 @@ jobs: test: strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - node: ["17.3"] + os: [ubuntu-latest] + node: ["16", "18", "20"] runs-on: ${{ matrix.os }} @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: '17.3' + node-version: ${{ matrix.node }} - run: npm install - run: npm test - run: npm run report -- --colors diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 7f0fa59..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,98 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) and this -project adheres to [Semantic Versioning](http://semver.org/). - -## v3.1.3 -- Allow usage of iterable object in Blob constructor. [#108] -- Run test WPT test against our impl [#109] -- File name are now casted to string [#109] -- Slicing in the middle of multiple parts added more bytes than what what it should have [#109] -- Prefixed `stream/web` import with `node:` to allow easier static analysis detection of Node built-ins [#122] -- Added `node:` prefix in `from.js` as well [#114] -- Suppress warning when importing `stream/web` [#114] - -## v3.1.2 -- Improved typing -- Fixed a bug where position in iterator did not increase - -## v3.1.0 -- started to use real whatwg streams -- degraded fs/promise to fs.promise to support node v12 -- degraded optional changing to support node v12 - -## v3.0.0 -- Changed WeakMap for private field (require node 12) -- Switch to ESM -- blob.stream() return a subset of whatwg stream which is the async iterable part - (it no longer return a node stream) -- Reduced the dependency of Buffer by changing to global TextEncoder/Decoder (require node 11) -- Disabled xo since it could understand private fields (#) -- No longer transform the type to lowercase (https://github.com/w3c/FileAPI/issues/43) - This is more loose than strict, keys should be lowercased, but values should not. - It would require a more proper mime type parser - so we just made it loose. -- index.js and file.js can now be imported by browser & deno since it no longer depends on any - core node features (but why would you?) -- Implemented a File class - -## v2.1.2 -- Fixed a bug where `start` in BlobDataItem was undefined (#85) - -## v2.1.1 -- Add nullish values checking in Symbol.hasInstance (#82) -- Add generated typings for from.js file (#80) -- Updated dev dependencies - -## v2.1.0 -- Fix: .slice has an implementation bug (#54). -- Added blob backed up by filesystem (#55) - -## v2.0.1 - -- Fix: remove upper bound for node engine semver (#49). - -## v2.0.0 - -> Note: This release was previously published as `1.0.7`, but as it contains breaking changes, we renamed it to `2.0.0`. - -- **Breaking:** minimum supported Node.js version is now 10.17. -- **Breaking:** `buffer` option has been removed. -- Enhance: create TypeScript declarations from JSDoc (#45). -- Enhance: operate on blob parts (byte sequence) (#44). -- Enhance: use a `WeakMap` for private properties (#42) . -- Other: update formatting. - -## v1.0.6 - -- Enhance: use upstream Blob directly in typings (#38) -- Other: update dependencies - -## v1.0.5 - -- Other: no change to code, update dev dependency to address vulnerability reports - -## v1.0.4 - -- Other: general code rewrite to pass linting, prepare for `node-fetch` release v3 - -## v1.0.3 - -- Fix: package.json export `blob.js` properly now - -## v1.0.2 - -- Other: fix test integration - -## v1.0.1 - -- Other: readme update - -## v1.0.0 - -- Major: initial release - -[#108]: https://github.com/node-fetch/fetch-blob/pull/108 -[#109]: https://github.com/node-fetch/fetch-blob/pull/109 -[#114]: https://github.com/node-fetch/fetch-blob/pull/114 diff --git a/README.md b/README.md index 1cd2d6a..7368a10 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,6 @@ npm install fetch-blob - CommonJS was replaced with ESM - The node stream returned by calling `blob.stream()` was replaced with whatwg streams - (Read "Differences from other blobs" for more info.) -
@@ -48,14 +47,10 @@ npm install fetch-blob ```js // Ways to import -// (PS it's dependency free ESM package so regular http-import from CDN works too) -import Blob from 'fetch-blob' -import File from 'fetch-blob/file.js' - -import {Blob} from 'fetch-blob' -import {File} from 'fetch-blob/file.js' +import { Blob } from 'fetch-blob' +import { File } from 'fetch-blob/file.js' -const {Blob} = await import('fetch-blob') +const { Blob } = await import('fetch-blob') // Ways to read the blob: @@ -75,7 +70,6 @@ It will not read the content into memory. It will only stat the file for last mo ```js // The default export is sync and use fs.stat to retrieve size & last modified as a blob -import blobFromSync from 'fetch-blob/from.js' import {File, Blob, blobFrom, blobFromSync, fileFrom, fileFromSync} from 'fetch-blob/from.js' const fsFile = fileFromSync('./2-GiB-file.bin', 'application/octet-stream') @@ -119,7 +113,8 @@ blob = undefined // loosing references will delete the file from disk ### Creating Blobs backed up by other async sources Our Blob & File class are more generic then any other polyfills in the way that it can accept any blob look-a-like item -An example of this is that our blob implementation can be constructed with parts coming from [BlobDataItem](https://github.com/node-fetch/fetch-blob/blob/8ef89adad40d255a3bbd55cf38b88597c1cd5480/from.js#L32) (aka a filepath) or from [buffer.Blob](https://nodejs.org/api/buffer.html#buffer_new_buffer_blob_sources_options), It dose not have to implement all the methods - just enough that it can be read/understood by our Blob implementation. The minium requirements is that it has `Symbol.toStringTag`, `size`, `slice()` and either a `stream()` or a `arrayBuffer()` method. If you then wrap it in our Blob or File `new Blob([blobDataItem])` then you get all of the other methods that should be implemented in a blob or file +An example of this is that our blob implementation can be constructed with parts coming from [BlobDataItem](https://github.com/node-fetch/fetch-blob/blob/8ef89adad40d255a3bbd55cf38b88597c1cd5480/from.js#L32) (aka a filepath) or from [buffer.Blob](https://nodejs.org/api/buffer.html#buffer_new_buffer_blob_sources_options), It dose not have to implement all the methods - just enough that it can be read/understood by our Blob implementation. The minium requirements is that it has `Symbol.toStringTag`, `size`, `slice()`, `stream()` methods (the stream method +can be as simple as being a sync or async iterator that yields Uint8Arrays. If you then wrap it in our Blob or File `new Blob([blobDataItem])` then you get all of the other methods that should be implemented in a blob or file (aka: text(), arrayBuffer() and type and a ReadableStream) An example of this could be to create a file or blob like item coming from a remote HTTP request. Or from a DataBase diff --git a/file.js b/file.js index 7b26538..b2739c9 100644 --- a/file.js +++ b/file.js @@ -1,4 +1,4 @@ -import Blob from './index.js' +import { Blob } from './index.js' const _File = class File extends Blob { #lastModified = 0 @@ -46,4 +46,3 @@ const _File = class File extends Blob { /** @type {typeof globalThis.File} */// @ts-ignore export const File = _File -export default File diff --git a/from.js b/from.js index cd0592a..d54d129 100644 --- a/from.js +++ b/from.js @@ -10,8 +10,8 @@ import { tmpdir } from 'node:os' import process from 'node:process' import DOMException from 'node-domexception' -import File from './file.js' -import Blob from './index.js' +import { File } from './file.js' +import { Blob } from './index.js' const { stat, mkdtemp } = fs let i = 0, tempDir, registry diff --git a/index.js b/index.js index 8a3809c..0683fa5 100644 --- a/index.js +++ b/index.js @@ -1,19 +1,32 @@ /*! fetch-blob. MIT License. Jimmy Wärting */ -// TODO (jimmywarting): in the feature use conditional loading with top level await (requires 14.x) -// Node has recently added whatwg stream into core - -import './streams.cjs' +if (!globalThis.ReadableStream) { + try { + const process = await import('node:process').then(m => m.default) + const { emitWarning } = process + try { + process.emitWarning = () => {} + const streams = await import('node:stream/web').then(m => m.default) + Object.assign(globalThis, streams) + process.emitWarning = emitWarning + } catch (error) { + process.emitWarning = emitWarning + throw error + } + } catch (error) {} +} // 64 KiB (same size chrome slice theirs blob into Uint8array's) const POOL_SIZE = 65536 -/** @param {(Blob | Uint8Array)[]} parts */ -async function * toIterator (parts, clone = true) { +/** + * @param {(Blob | Uint8Array)[]} parts + * @param {boolean} clone + * @returns {AsyncIterableIterator} + */ +async function * toIterator (parts, clone) { for (const part of parts) { - if ('stream' in part) { - yield * (/** @type {AsyncIterableIterator} */ (part.stream())) - } else if (ArrayBuffer.isView(part)) { + if (ArrayBuffer.isView(part)) { if (clone) { let position = part.byteOffset const end = part.byteOffset + part.byteLength @@ -26,16 +39,9 @@ async function * toIterator (parts, clone = true) { } else { yield part } - /* c8 ignore next 10 */ } else { - // For blobs that have arrayBuffer but no stream method (nodes buffer.Blob) - let position = 0, b = (/** @type {Blob} */ (part)) - while (position !== b.size) { - const chunk = b.slice(position, Math.min(b.size, position + POOL_SIZE)) - const buffer = await chunk.arrayBuffer() - position += buffer.byteLength - yield new Uint8Array(buffer) - } + // @ts-ignore TS Think blob.stream() returns a node:stream + yield * part.stream() } } } @@ -139,11 +145,6 @@ const _Blob = class Blob { * @return {Promise} */ async arrayBuffer () { - // Easier way... Just a unnecessary overhead - // const view = new Uint8Array(this.size); - // await this.stream().getReader({mode: 'byob'}).read(view); - // return view.buffer; - const data = new Uint8Array(this.size) let offset = 0 for await (const chunk of toIterator(this.#parts, false)) { @@ -218,7 +219,7 @@ const _Blob = class Blob { } } - const blob = new Blob([], { type: String(type).toLowerCase() }) + const blob = new Blob([], { type: `${type}` }) blob.#size = span blob.#parts = blobParts @@ -251,4 +252,3 @@ Object.defineProperties(_Blob.prototype, { /** @type {typeof globalThis.Blob} */ export const Blob = _Blob -export default Blob diff --git a/package.json b/package.json index 718ddae..2a8fbd5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fetch-blob", - "version": "3.1.5", + "version": "4.0.0", "description": "Blob & File implementation in Node.js, originally from node-fetch.", "main": "index.js", "type": "module", @@ -10,8 +10,7 @@ "file.d.ts", "index.js", "index.d.ts", - "from.d.ts", - "streams.cjs" + "from.d.ts" ], "scripts": { "test": "node --experimental-loader ./test/http-loader.js ./test/test-wpt-in-node.js", @@ -26,7 +25,7 @@ "node-fetch" ], "engines": { - "node": "^12.20 || >= 14.13" + "node": ">=16.7" }, "author": "Jimmy Wärting (https://jimmy.warting.se)", "license": "MIT", @@ -35,9 +34,9 @@ }, "homepage": "https://github.com/node-fetch/fetch-blob#readme", "devDependencies": { - "@types/node": "^18.0.2", - "c8": "^7.11.0", - "typescript": "^4.5.4" + "@types/node": "^16.5.0", + "c8": "^7.13.0", + "typescript": "^5.0.4" }, "funding": [ { @@ -50,7 +49,6 @@ } ], "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" + "node-domexception": "^1.0.0" } } diff --git a/streams.cjs b/streams.cjs deleted file mode 100644 index f760959..0000000 --- a/streams.cjs +++ /dev/null @@ -1,51 +0,0 @@ -/* c8 ignore start */ -// 64 KiB (same size chrome slice theirs blob into Uint8array's) -const POOL_SIZE = 65536 - -if (!globalThis.ReadableStream) { - // `node:stream/web` got introduced in v16.5.0 as experimental - // and it's preferred over the polyfilled version. So we also - // suppress the warning that gets emitted by NodeJS for using it. - try { - const process = require('node:process') - const { emitWarning } = process - try { - process.emitWarning = () => {} - Object.assign(globalThis, require('node:stream/web')) - process.emitWarning = emitWarning - } catch (error) { - process.emitWarning = emitWarning - throw error - } - } catch (error) { - // fallback to polyfill implementation - Object.assign(globalThis, require('web-streams-polyfill/dist/ponyfill.es2018.js')) - } -} - -try { - // Don't use node: prefix for this, require+node: is not supported until node v14.14 - // Only `import()` can use prefix in 12.20 and later - const { Blob } = require('buffer') - if (Blob && !Blob.prototype.stream) { - Blob.prototype.stream = function name (params) { - let position = 0 - const blob = this - - return new ReadableStream({ - type: 'bytes', - async pull (ctrl) { - const chunk = blob.slice(position, Math.min(blob.size, position + POOL_SIZE)) - const buffer = await chunk.arrayBuffer() - position += buffer.byteLength - ctrl.enqueue(new Uint8Array(buffer)) - - if (position === blob.size) { - ctrl.close() - } - } - }) - } - } -} catch (error) {} -/* c8 ignore end */ diff --git a/test/http-loader.js b/test/http-loader.js index cbbdbf0..5910a7a 100644 --- a/test/http-loader.js +++ b/test/http-loader.js @@ -5,36 +5,29 @@ import { get } from 'node:https' const fetch = url => new Promise(rs => get(url, rs)) const cache = new URL('./.cache/', import.meta.url) -/** - * @param {string} specifier - * @param {{ - * conditions: !Array, - * parentURL: !(string | undefined), - * }} context - * @param {Function} defaultResolve - * @returns {Promise<{ url: string }>} - */ -export async function resolve (specifier, context, defaultResolve) { - const { parentURL = null } = context +export function resolve(specifier, context, nextResolve) { + const { parentURL = null } = context; // Normally Node.js would error on specifiers starting with 'https://', so // this hook intercepts them and converts them into absolute URLs to be // passed along to the later hooks below. if (specifier.startsWith('https://')) { return { - url: specifier - } + shortCircuit: true, + url: specifier, + }; } else if (parentURL && parentURL.startsWith('https://')) { return { - url: new URL(specifier, parentURL).href - } + shortCircuit: true, + url: new URL(specifier, parentURL).href, + }; } // Let Node.js handle all other specifiers. - return defaultResolve(specifier, context, defaultResolve) + return nextResolve(specifier); } -export async function load (url, context, defaultLoad) { +export async function load(url, context, nextLoad) { // For JavaScript to be loaded over the network, we need to fetch and // return it. if (url.startsWith('https://')) { @@ -52,14 +45,15 @@ export async function load (url, context, defaultLoad) { fs.writeFileSync(cachedFile, data) } + // This example assumes all network-provided JavaScript is ES module + // code. return { - // This example assumes all network-provided JavaScript is ES module - // code. format: 'module', - source: data + shortCircuit: true, + source: data, } } // Let Node.js handle all other URLs. - return defaultLoad(url, context, defaultLoad) -} + return nextLoad(url); +} \ No newline at end of file diff --git a/test/own-misc-test.js b/test/own-misc-test.js index 8189100..b1336c9 100644 --- a/test/own-misc-test.js +++ b/test/own-misc-test.js @@ -63,11 +63,6 @@ test(() => { assert_equals(blobFromSync, syncBlob) }, 'default export is named exported blobFromSync') -promise_test(async () => { - const { Blob, default: def } = await import('../index.js') - assert_equals(Blob, def) -}, 'Can use named import - as well as default') - // This was necessary to avoid large ArrayBuffer clones (slice) promise_test(async t => { const buf = new Uint8Array(65590) diff --git a/test/test-wpt-in-node.js b/test/test-wpt-in-node.js index e414298..96f06f5 100644 --- a/test/test-wpt-in-node.js +++ b/test/test-wpt-in-node.js @@ -22,7 +22,7 @@ function test_blob (fn, expectations) { const blob = fn() assert_true(blob instanceof Blob) assert_false(blob instanceof File) - assert_equals(blob.type.toLowerCase(), type) + assert_equals(blob.type.toLowerCase(), type.toLowerCase()) assert_equals(await blob.text(), expected) t.done() }) @@ -140,4 +140,7 @@ import('https://wpt.live/FileAPI/blob/Blob-stream.any.js') import('https://wpt.live/FileAPI/blob/Blob-text.any.js') import('./own-misc-test.js') -hasFailed && process.exit(1) \ No newline at end of file +if (hasFailed) { + console.log('Tests failed') + process.exit(1) +} \ No newline at end of file