fix: more Tsize fixes, fix HAMT and make it match go-unixfs output

Author: rvagg

data/builder/dir_test.go

@@ -45,13 +45,11 @@ func TestBuildUnixFSDirectory(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		dl, tsize, err := BuildUnixFSDirectory(entries, &ls)
+		dl, _, err := BuildUnixFSDirectory(entries, &ls)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		require.GreaterOrEqual(t, tsize, uint64(0)) // TODO: set properly
-
 		pbn, err := ls.Load(ipld.LinkContext{}, dl, dagpb.Type.PBNode)
 		if err != nil {
 			t.Fatal(err)
@@ -108,8 +106,36 @@ func TestBuildUnixFSRecursive(t *testing.T) {
 
 	lnk, sz, err := BuildUnixFSRecursive(filepath.Join(dir, fixture.name), &ls)
 	require.NoError(t, err)
-	require.Equal(t, lnk.String(), fixture.expectedLnk.String())
-	require.Equal(t, sz, uint64(245))
+	require.Equal(t, fixture.expectedLnk.String(), lnk.String())
+	require.Equal(t, uint64(245), sz)
+}
+
+func TestBuildUnixFSRecursiveSharded(t *testing.T) {
+	// only the top CID is of interest, but this tree is correct and can be used for future validation
+	fixture := fentry{
+		"rootDir",
+		"",
+		mustCidDecode("bafybeiendaawtta62lx2p2e2hecgywmqeq6ekrn2pfypxjkmdzmaeituhe"),
+		make([]fentry, 0),
+	}
+
+	for i := 0; i < 2048; i++ {
+		name := fmt.Sprintf("long name to fill out bytes to make the sharded directory test flip over the sharded directory limit because link names are included in the directory entry %d", i)
+		fixture.children = append(fixture.children, fentry{name, name, cid.Undef, nil})
+	}
+
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	dir := t.TempDir()
+	makeFixture(t, dir, fixture)
+
+	lnk, sz, err := BuildUnixFSRecursive(filepath.Join(dir, fixture.name), &ls)
+	require.NoError(t, err)
+	require.Equal(t, fixture.expectedLnk.String(), lnk.String())
+	require.Equal(t, uint64(778128), sz)
 }
 
 type fentry struct {

data/builder/directory.go

@@ -48,8 +48,7 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64,
 			}
 			lnks = append(lnks, entry)
 		}
-		outLnk, sz, err := BuildUnixFSDirectory(lnks, ls)
-		return outLnk, tsize + sz, err
+		return BuildUnixFSDirectory(lnks, ls)
 	case m.Type() == fs.ModeSymlink:
 		content, err := os.Readlink(root)
 		if err != nil {
@@ -126,7 +125,9 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin
 		return nil, 0, err
 	}
 	// sorting happens in codec-dagpb
+	var totalSize uint64
 	for _, e := range entries {
+		totalSize += uint64(e.Tsize.Must().Int())
 		if err := lnks.AssembleValue().AssignNode(e); err != nil {
 			return nil, 0, err
 		}
@@ -138,5 +139,9 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin
 		return nil, 0, err
 	}
 	node := pbb.Build()
-	return sizedStore(ls, fileLinkProto, node)
+	lnk, sz, err := sizedStore(ls, fileLinkProto, node)
+	if err != nil {
+		return nil, 0, err
+	}
+	return lnk, totalSize + sz, err
 }

data/builder/dirshard.go

@@ -56,7 +56,9 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink
 	hamtEntries := make([]hamtLink, 0, len(entries))
 	for _, e := range entries {
 		name := e.Name.Must().String()
-		sum := h.Sum([]byte(name))
+		h.Reset()
+		h.Write([]byte(name))
+		sum := h.Sum(nil)
 		hamtEntries = append(hamtEntries, hamtLink{
 			sum,
 			e,
@@ -97,9 +99,11 @@ func (s *shard) add(lnk hamtLink) error {
 
 	current, ok := s.children[bucket]
 	if !ok {
+		// no bucket, make one with this entry
 		s.children[bucket] = entry{nil, &lnk}
 		return nil
 	} else if current.shard != nil {
+		// existing shard, add this link to the shard
 		return current.shard.add(lnk)
 	}
 	// make a shard for current and lnk
@@ -114,15 +118,18 @@ func (s *shard) add(lnk hamtLink) error {
 		},
 		nil,
 	}
+	// add existing link from this bucket to the new shard
 	if err := newShard.add(*current.hamtLink); err != nil {
 		return err
 	}
+	// replace bucket with shard
 	s.children[bucket] = newShard
+	// add new link to the new shard
 	return newShard.add(lnk)
 }
 
 func (s *shard) formatLinkName(name string, idx int) string {
-	return fmt.Sprintf("%*X%s", s.width, idx, name)
+	return fmt.Sprintf("%0*X%s", s.width, idx, name)
 }
 
 // bitmap calculates the bitmap of which links in the shard are set.
@@ -169,21 +176,25 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, uint64, error) {
 		return nil, 0, err
 	}
 	// sorting happens in codec-dagpb
+	var totalSize uint64
 	for idx, e := range s.children {
 		var lnk dagpb.PBLink
 		if e.shard != nil {
 			ipldLnk, sz, err := e.shard.serialize(ls)
 			if err != nil {
 				return nil, 0, err
 			}
+			totalSize += sz
 			fullName := s.formatLinkName("", idx)
 			lnk, err = BuildUnixFSDirectoryEntry(fullName, int64(sz), ipldLnk)
 			if err != nil {
 				return nil, 0, err
 			}
 		} else {
 			fullName := s.formatLinkName(e.Name.Must().String(), idx)
-			lnk, err = BuildUnixFSDirectoryEntry(fullName, e.Tsize.Must().Int(), e.Hash.Link())
+			sz := e.Tsize.Must().Int()
+			totalSize += uint64(sz)
+			lnk, err = BuildUnixFSDirectoryEntry(fullName, sz, e.Hash.Link())
 		}
 		if err != nil {
 			return nil, 0, err
@@ -200,5 +211,9 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, uint64, error) {
 		return nil, 0, err
 	}
 	node := pbb.Build()
-	return sizedStore(ls, fileLinkProto, node)
+	lnk, sz, err := sizedStore(ls, fileLinkProto, node)
+	if err != nil {
+		return nil, 0, err
+	}
+	return lnk, totalSize + sz, nil
 }