From b25790027771e8b1970be432d52b5897a0812b0f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 10 Mar 2023 22:03:30 +0100 Subject: [PATCH] fix: use simpler blockstore interface Instead of expecting a full `interface-blockstore` implementation, only use a subset of the methods. --- packages/ipfs-unixfs-exporter/package.json | 4 ++-- packages/ipfs-unixfs-exporter/src/index.ts | 8 +++++++- .../src/resolvers/unixfs-v1/content/file.ts | 3 +-- .../unixfs-v1/content/hamt-sharded-directory.ts | 12 +----------- .../src/utils/find-cid-in-shard.ts | 3 +-- packages/ipfs-unixfs-importer/package.json | 4 ++-- packages/ipfs-unixfs-importer/src/dag-builder/dir.ts | 3 +-- .../ipfs-unixfs-importer/src/dag-builder/file.ts | 7 +++---- .../ipfs-unixfs-importer/src/dag-builder/index.ts | 3 +-- packages/ipfs-unixfs-importer/src/dir.ts | 3 +-- packages/ipfs-unixfs-importer/src/index.ts | 8 +++++++- packages/ipfs-unixfs-importer/src/tree-builder.ts | 3 +-- packages/ipfs-unixfs-importer/src/utils/persist.ts | 2 +- 13 files changed, 29 insertions(+), 34 deletions(-) diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 6a0cb1be..6cb35a12 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -131,7 +131,7 @@ "build": "aegir build", "clean": "aegir clean", "lint": "aegir lint", - "dep-check": "aegir dep-check -i interface-blockstore", + "dep-check": "aegir dep-check", "release": "aegir release" }, "dependencies": { @@ -140,7 +140,7 @@ "@multiformats/murmur3": "^2.0.0", "err-code": "^3.0.1", "hamt-sharding": "^3.0.0", - "interface-blockstore": "^4.0.0", + "interface-blockstore": "^4.0.1", "ipfs-unixfs": "^11.0.0", "it-filter": "^2.0.0", "it-last": "^2.0.0", diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index 8c3b984b..4528ece3 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -4,7 +4,7 @@ import resolve from './resolvers/index.js' import last from 'it-last' import type { UnixFS } from 'ipfs-unixfs' import type { PBNode } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore' import type { Bucket } from 'hamt-sharding' export interface ExporterOptions { @@ -79,6 +79,12 @@ export interface ShardTraversalContext { lastBucket: Bucket } +export interface BlockstoreOptions { + signal?: AbortSignal +} + +export type Blockstore = Pick + const toPathComponents = (path: string = ''): string[] => { // split on / unless escaped with \ return (path diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index b3798226..191d97b1 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -9,8 +9,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import map from 'it-map' import PQueue from 'p-queue' -import type { Blockstore } from 'interface-blockstore' -import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver } from '../../../index.js' +import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, Blockstore } from '../../../index.js' async function walkDAG (blockstore: Blockstore, node: dagPb.PBNode | Uint8Array, queue: Pushable, streamPosition: bigint, start: bigint, end: bigint, walkQueue: PQueue, options: ExporterOptions): Promise { // a `raw` node diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index 203a3448..0d5086fe 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -2,17 +2,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import map from 'it-map' import { decode, PBNode } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js' - -/** - * @typedef {import('interface-blockstore').Blockstore} Blockstore - * @typedef {import('../../../types').ExporterOptions} ExporterOptions - * @typedef {import('../../../types').Resolve} Resolve - * @typedef {import('../../../types').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent - * @typedef {import('../../../types').UnixfsV1Resolver} UnixfsV1Resolver - * @typedef {import('@ipld/dag-pb').PBNode} PBNode - */ +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, Blockstore } from '../../../index.js' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts index a0e36ee1..44ecfe22 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts @@ -2,8 +2,7 @@ import { Bucket, BucketPosition, createHAMT } from 'hamt-sharding' import { decode, PBLink, PBNode } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' -import type { Blockstore } from 'interface-blockstore' -import type { ExporterOptions, ShardTraversalContext } from '../index.js' +import type { ExporterOptions, ShardTraversalContext, Blockstore } from '../index.js' import type { CID } from 'multiformats/cid' // FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index bc434f68..8385313a 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -155,7 +155,7 @@ "build": "aegir build", "clean": "aegir clean", "lint": "aegir lint", - "dep-check": "aegir dep-check -i interface-blockstore", + "dep-check": "aegir dep-check", "release": "aegir release" }, "dependencies": { @@ -163,8 +163,8 @@ "@multiformats/murmur3": "^2.0.0", "err-code": "^3.0.1", "hamt-sharding": "^3.0.0", - "interface-blockstore": "^4.0.0", "ipfs-unixfs": "^11.0.0", + "interface-blockstore": "^4.0.1", "it-all": "^2.0.0", "it-batch": "^2.0.0", "it-first": "^2.0.0", diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts index 21c4ccae..b367119b 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts @@ -1,8 +1,7 @@ import { UnixFS } from 'ipfs-unixfs' import { persist } from '../utils/persist.js' import { encode, prepare } from '@ipld/dag-pb' -import type { Directory, InProgressImportResult } from '../index.js' -import type { Blockstore } from 'interface-blockstore' +import type { Directory, InProgressImportResult, Blockstore } from '../index.js' import type { Version } from 'multiformats/cid' export interface DirBuilderOptions { diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 268c1667..6329ecec 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -3,8 +3,7 @@ import { persist } from '../utils/persist.js' import { encode, PBLink, prepare } from '@ipld/dag-pb' import parallelBatch from 'it-parallel-batch' import * as rawCodec from 'multiformats/codecs/raw' -import type { BufferImporter, File, InProgressImportResult } from '../index.js' -import type { Blockstore } from 'interface-blockstore' +import type { BufferImporter, File, InProgressImportResult, Blockstore } from '../index.js' import type { FileLayout, Reducer } from '../layout/index.js' import type { Version } from 'multiformats/cid' @@ -49,9 +48,9 @@ const reduce = (file: File, blockstore: Blockstore, options: ReduceOptions): Red const leaf = leaves[0] if (file.mtime !== undefined || file.mode !== undefined) { - // only one leaf node which is a buffer - we have metadata so convert it into a + // only one leaf node which is a raw leaf - we have metadata so convert it into a // UnixFS entry otherwise we'll have nowhere to store the metadata - let buffer = await blockstore.get(leaf.cid) + let buffer = await blockstore.get(leaf.cid, options) leaf.unixfs = new UnixFS({ type: 'file', diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 3ef84010..3779cc7b 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,8 +1,7 @@ import { dirBuilder, DirBuilderOptions } from './dir.js' import { fileBuilder, FileBuilderOptions } from './file.js' import errCode from 'err-code' -import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult } from '../index.js' -import type { Blockstore } from 'interface-blockstore' +import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult, Blockstore } from '../index.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' diff --git a/packages/ipfs-unixfs-importer/src/dir.ts b/packages/ipfs-unixfs-importer/src/dir.ts index 1e0303d7..8bf65583 100644 --- a/packages/ipfs-unixfs-importer/src/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dir.ts @@ -1,7 +1,6 @@ -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore, ImportResult, InProgressImportResult } from './index.js' import type { Mtime, UnixFS } from 'ipfs-unixfs' import { CID } from 'multiformats/cid' -import type { ImportResult, InProgressImportResult } from './index.js' import type { PersistOptions } from './utils/persist.js' export interface DirProps { diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 691c05f1..fb33df35 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -3,7 +3,7 @@ import { DAGBuilder, defaultDagBuilder } from './dag-builder/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { UnixFS, Mtime } from 'ipfs-unixfs' import type { CID, Version as CIDVersion } from 'multiformats/cid' -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore' import { ChunkValidator, defaultChunkValidator } from './dag-builder/validate-chunks.js' import { fixedSize } from './chunker/fixed-size.js' import type { Chunker } from './chunker/index.js' @@ -16,6 +16,12 @@ import type { AwaitIterable } from 'blockstore-core/base' export type ByteStream = AwaitIterable export type ImportContent = ByteStream | Uint8Array +export interface BlockstoreOptions { + signal?: AbortSignal +} + +export type Blockstore = Pick + export interface FileCandidate { path?: string content: ImportContent diff --git a/packages/ipfs-unixfs-importer/src/tree-builder.ts b/packages/ipfs-unixfs-importer/src/tree-builder.ts index b40e0f29..2b5946a5 100644 --- a/packages/ipfs-unixfs-importer/src/tree-builder.ts +++ b/packages/ipfs-unixfs-importer/src/tree-builder.ts @@ -2,8 +2,7 @@ import { DirFlat } from './dir-flat.js' import { flatToShard } from './flat-to-shard.js' import { Dir } from './dir.js' import { toPathComponents } from './utils/to-path-components.js' -import type { ImportResult, InProgressImportResult, TreeBuilder } from './index.js' -import type { Blockstore } from 'interface-blockstore' +import type { ImportResult, InProgressImportResult, TreeBuilder, Blockstore } from './index.js' import type { PersistOptions } from './utils/persist.js' export interface AddToTreeOptions extends PersistOptions { diff --git a/packages/ipfs-unixfs-importer/src/utils/persist.ts b/packages/ipfs-unixfs-importer/src/utils/persist.ts index 15f12752..06b6557b 100644 --- a/packages/ipfs-unixfs-importer/src/utils/persist.ts +++ b/packages/ipfs-unixfs-importer/src/utils/persist.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import { sha256 } from 'multiformats/hashes/sha2' -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore } from '../index.js' import type { BlockCodec } from 'multiformats/codecs/interface' import type { Version as CIDVersion } from 'multiformats/cid'