Skip to content

Commit 9693f3f

Browse files
authored
fix: use fanout "bits" (#357)
1 parent d76a5d2 commit 9693f3f

File tree

4 files changed

+13
-9
lines changed

4 files changed

+13
-9
lines changed

packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ describe('exporter sharded', function () {
270270

271271
const result = await last(importer(files, block, {
272272
shardSplitThresholdBytes: 0,
273-
shardFanoutBytes: 4,
273+
shardFanoutBits: 4, // 2**4 = 16 children max
274274
wrapWithDirectory: true
275275
}))
276276

packages/ipfs-unixfs-importer/src/dir-sharded.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@ async function hamtHashFn (buf: Uint8Array): Promise<Uint8Array> {
1818
}
1919

2020
const HAMT_HASH_CODE = BigInt(0x22)
21+
const DEFAULT_FANOUT_BITS = 8
2122

2223
export interface DirShardedOptions extends PersistOptions {
23-
shardFanoutBytes: number
24+
shardFanoutBits: number
2425
}
2526

2627
class DirSharded extends Dir {
@@ -31,7 +32,7 @@ class DirSharded extends Dir {
3132

3233
this._bucket = createHAMT({
3334
hashFn: hamtHashFn,
34-
bits: options.shardFanoutBytes ?? 8
35+
bits: options.shardFanoutBits ?? DEFAULT_FANOUT_BITS
3536
})
3637
}
3738

@@ -196,6 +197,7 @@ function isDir (obj: any): obj is Dir {
196197

197198
function calculateSize (bucket: Bucket<any>, shardRoot: DirSharded | null, options: PersistOptions): number {
198199
const children = bucket._children
200+
const padLength = (bucket.tableSize() - 1).toString(16).length
199201
const links: PBLink[] = []
200202

201203
for (let i = 0; i < children.length; i++) {
@@ -205,7 +207,7 @@ function calculateSize (bucket: Bucket<any>, shardRoot: DirSharded | null, optio
205207
continue
206208
}
207209

208-
const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
210+
const labelPrefix = i.toString(16).toUpperCase().padStart(padLength, '0')
209211

210212
if (child instanceof Bucket) {
211213
const size = calculateSize(child, null, options)

packages/ipfs-unixfs-importer/src/index.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,11 @@ export interface ImporterOptions extends ProgressOptions<ImporterProgressEvents>
124124
shardSplitThresholdBytes?: number
125125

126126
/**
127-
* The maximum number of bytes used as a HAMT prefix for shard entries. Default: 256
127+
* The number of bits of a hash digest used at each level of sharding to
128+
* the child index. 2**shardFanoutBits will dictate the maximum number of
129+
* children for any shard in the HAMT. Default: 8
128130
*/
129-
shardFanoutBytes?: number
131+
shardFanoutBits?: number
130132

131133
/**
132134
* How many files to import concurrently. For large numbers of small files this
@@ -246,7 +248,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri
246248

247249
const wrapWithDirectory = options.wrapWithDirectory ?? false
248250
const shardSplitThresholdBytes = options.shardSplitThresholdBytes ?? 262144
249-
const shardFanoutBytes = options.shardFanoutBytes ?? 8
251+
const shardFanoutBits = options.shardFanoutBits ?? 8
250252
const cidVersion = options.cidVersion ?? 1
251253
const rawLeaves = options.rawLeaves ?? true
252254
const leafType = options.leafType ?? 'file'
@@ -275,7 +277,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri
275277
const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({
276278
wrapWithDirectory,
277279
shardSplitThresholdBytes,
278-
shardFanoutBytes,
280+
shardFanoutBits,
279281
cidVersion,
280282
onProgress: options.onProgress
281283
})

packages/ipfs-unixfs-importer/src/tree-builder.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import type { PersistOptions } from './utils/persist.js'
77

88
export interface AddToTreeOptions extends PersistOptions {
99
shardSplitThresholdBytes: number
10-
shardFanoutBytes: number
10+
shardFanoutBits: number
1111
}
1212

1313
async function addToTree (elem: InProgressImportResult, tree: Dir, options: AddToTreeOptions): Promise<Dir> {

0 commit comments

Comments
 (0)