From 61e5755331144c55b1548681d4d9f419ed72269d Mon Sep 17 00:00:00 2001
From: Rod Vagg <rod@vagg.org>
Date: Fri, 25 Aug 2023 16:05:28 +1000
Subject: [PATCH] fix: use fanout "bits"

---
 .../ipfs-unixfs-exporter/test/exporter-sharded.spec.ts |  2 +-
 packages/ipfs-unixfs-importer/src/dir-sharded.ts       |  8 +++++---
 packages/ipfs-unixfs-importer/src/index.ts             | 10 ++++++----
 packages/ipfs-unixfs-importer/src/tree-builder.ts      |  2 +-
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts
index 9da7a83d..95a8c50e 100644
--- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts
+++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts
@@ -270,7 +270,7 @@ describe('exporter sharded', function () {
 
     const result = await last(importer(files, block, {
       shardSplitThresholdBytes: 0,
-      shardFanoutBytes: 4,
+      shardFanoutBits: 4, // 2**4 = 16 children max
       wrapWithDirectory: true
     }))
 
diff --git a/packages/ipfs-unixfs-importer/src/dir-sharded.ts b/packages/ipfs-unixfs-importer/src/dir-sharded.ts
index 7b0914dd..7bee99b0 100644
--- a/packages/ipfs-unixfs-importer/src/dir-sharded.ts
+++ b/packages/ipfs-unixfs-importer/src/dir-sharded.ts
@@ -18,9 +18,10 @@ async function hamtHashFn (buf: Uint8Array): Promise<Uint8Array> {
 }
 
 const HAMT_HASH_CODE = BigInt(0x22)
+const DEFAULT_FANOUT_BITS = 8
 
 export interface DirShardedOptions extends PersistOptions {
-  shardFanoutBytes: number
+  shardFanoutBits: number
 }
 
 class DirSharded extends Dir {
@@ -31,7 +32,7 @@ class DirSharded extends Dir {
 
     this._bucket = createHAMT({
       hashFn: hamtHashFn,
-      bits: options.shardFanoutBytes ?? 8
+      bits: options.shardFanoutBits ?? DEFAULT_FANOUT_BITS
     })
   }
 
@@ -196,6 +197,7 @@ function isDir (obj: any): obj is Dir {
 
 function calculateSize (bucket: Bucket<any>, shardRoot: DirSharded | null, options: PersistOptions): number {
   const children = bucket._children
+  const padLength = (bucket.tableSize() - 1).toString(16).length
   const links: PBLink[] = []
 
   for (let i = 0; i < children.length; i++) {
@@ -205,7 +207,7 @@ function calculateSize (bucket: Bucket<any>, shardRoot: DirSharded | null, optio
       continue
     }
 
-    const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+    const labelPrefix = i.toString(16).toUpperCase().padStart(padLength, '0')
 
     if (child instanceof Bucket) {
       const size = calculateSize(child, null, options)
diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts
index 5db54f81..d0dc4a7a 100644
--- a/packages/ipfs-unixfs-importer/src/index.ts
+++ b/packages/ipfs-unixfs-importer/src/index.ts
@@ -124,9 +124,11 @@ export interface ImporterOptions extends ProgressOptions<ImporterProgressEvents>
   shardSplitThresholdBytes?: number
 
   /**
-   * The maximum number of bytes used as a HAMT prefix for shard entries. Default: 256
+   * The number of bits of a hash digest used at each level of sharding to
+   * the child index. 2**shardFanoutBits will dictate the maximum number of
+   * children for any shard in the HAMT. Default: 8
    */
-  shardFanoutBytes?: number
+  shardFanoutBits?: number
 
   /**
    * How many files to import concurrently. For large numbers of small files this
@@ -246,7 +248,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri
 
   const wrapWithDirectory = options.wrapWithDirectory ?? false
   const shardSplitThresholdBytes = options.shardSplitThresholdBytes ?? 262144
-  const shardFanoutBytes = options.shardFanoutBytes ?? 8
+  const shardFanoutBits = options.shardFanoutBits ?? 8
   const cidVersion = options.cidVersion ?? 1
   const rawLeaves = options.rawLeaves ?? true
   const leafType = options.leafType ?? 'file'
@@ -275,7 +277,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri
   const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({
     wrapWithDirectory,
     shardSplitThresholdBytes,
-    shardFanoutBytes,
+    shardFanoutBits,
     cidVersion,
     onProgress: options.onProgress
   })
diff --git a/packages/ipfs-unixfs-importer/src/tree-builder.ts b/packages/ipfs-unixfs-importer/src/tree-builder.ts
index 674775e1..f84d6fb5 100644
--- a/packages/ipfs-unixfs-importer/src/tree-builder.ts
+++ b/packages/ipfs-unixfs-importer/src/tree-builder.ts
@@ -7,7 +7,7 @@ import type { PersistOptions } from './utils/persist.js'
 
 export interface AddToTreeOptions extends PersistOptions {
   shardSplitThresholdBytes: number
-  shardFanoutBytes: number
+  shardFanoutBits: number
 }
 
 async function addToTree (elem: InProgressImportResult, tree: Dir, options: AddToTreeOptions): Promise<Dir> {