Fix sharding memory growth, and fix resolver for unixfs paths

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

core/coreunix/cat.go

@@ -9,7 +9,12 @@ import (
 )
 
 func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (uio.DagReader, error) {
-	dagNode, err := core.Resolve(ctx, n.Namesys, n.Resolver, path.Path(pstr))
+	r := &path.Resolver{
+		DAG:         n.DAG,
+		ResolveOnce: uio.ResolveUnixfsOnce,
+	}
+
+	dagNode, err := core.Resolve(ctx, n.Namesys, r, path.Path(pstr))
 	if err != nil {
 		return nil, err
 	}

path/resolver.go

@@ -163,7 +163,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri
 		ctx, cancel = context.WithTimeout(ctx, time.Minute)
 		defer cancel()
 
-		lnk, rest, err := nd.ResolveLink(names)
+		lnk, err := s.ResolveOnce(ctx, s.DAG, nd, names[0])
 		if err == dag.ErrLinkNotFound {
 			return result, ErrNoLink{Name: names[0], Node: nd.Cid()}
 		} else if err != nil {
@@ -177,7 +177,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri
 
 		nd = nextnode
 		result = append(result, nextnode)
-		names = rest
+		names = names[1:]
 	}
 	return result, nil
 }

test/sharness/t0260-sharding-flag.sh

@@ -68,6 +68,11 @@ test_add_large_dir_v1() {
 		echo "$exphash" > sharddir_exp &&
 		test_cmp sharddir_exp sharddir_out
 	'
+
+	test_expect_success "can access a path under the dir" '
+		ipfs cat "$exphash/file20" > file20_out &&
+		test_cmp testdata/file20 file20_out
+	'
 }
 
 # this hash implies both the directory and the leaf entries are CIDv1

unixfs/hamt/hamt.go

@@ -62,7 +62,7 @@ type HamtShard struct {
 
 // child can either be another shard, or a leaf node value
 type child interface {
-	Node() (node.Node, error)
+	Link() (*node.Link, error)
 	Label() string
 }
 
@@ -144,12 +144,12 @@ func (ds *HamtShard) Node() (node.Node, error) {
 		cindex := ds.indexForBitPos(i)
 		ch := ds.children[cindex]
 		if ch != nil {
-			cnd, err := ch.Node()
+			clnk, err := ch.Link()
 			if err != nil {
 				return nil, err
 			}
 
-			err = out.AddNodeLinkClean(ds.linkNamePrefix(i)+ch.Label(), cnd)
+			err = out.AddRawLink(ds.linkNamePrefix(i)+ch.Label(), clnk)
 			if err != nil {
 				return nil, err
 			}
@@ -188,10 +188,10 @@ func (ds *HamtShard) Node() (node.Node, error) {
 
 type shardValue struct {
 	key string
-	val node.Node
+	val *node.Link
 }
 
-func (sv *shardValue) Node() (node.Node, error) {
+func (sv *shardValue) Link() (*node.Link, error) {
 	return sv.val, nil
 }
 
@@ -214,7 +214,18 @@ func (ds *HamtShard) Label() string {
 // Set sets 'name' = nd in the HAMT
 func (ds *HamtShard) Set(ctx context.Context, name string, nd node.Node) error {
 	hv := &hashBits{b: hash([]byte(name))}
-	return ds.modifyValue(ctx, hv, name, nd)
+	_, err := ds.dserv.Add(nd)
+	if err != nil {
+		return err
+	}
+
+	lnk, err := node.MakeLink(nd)
+	if err != nil {
+		return err
+	}
+	lnk.Name = ds.linkNamePrefix(0) + name
+
+	return ds.modifyValue(ctx, hv, name, lnk)
 }
 
 // Remove deletes the named entry if it exists, this operation is idempotent.
@@ -226,13 +237,16 @@ func (ds *HamtShard) Remove(ctx context.Context, name string) error {
 func (ds *HamtShard) Find(ctx context.Context, name string) (node.Node, error) {
 	hv := &hashBits{b: hash([]byte(name))}
 
-	var out node.Node
+	var out *node.Link
 	err := ds.getValue(ctx, hv, name, func(sv *shardValue) error {
 		out = sv.val
 		return nil
 	})
+	if err != nil {
+		return nil, err
+	}
 
-	return out, err
+	return ds.dserv.Get(ctx, out.Cid)
 }
 
 // getChild returns the i'th child of this shard. If it is cached in the
@@ -291,9 +305,10 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
 
 		c = cds
 	} else {
+		lnk2 := *lnk
 		c = &shardValue{
 			key: lnk.Name[ds.maxpadlen:],
-			val: nd,
+			val: &lnk2,
 		}
 	}
 
@@ -305,16 +320,32 @@ func (ds *HamtShard) setChild(i int, c child) {
 	ds.children[i] = c
 }
 
-func (ds *HamtShard) insertChild(idx int, key string, val node.Node) error {
-	if val == nil {
+func (ds *HamtShard) Link() (*node.Link, error) {
+	nd, err := ds.Node()
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = ds.dserv.Add(nd)
+	if err != nil {
+		return nil, err
+	}
+
+	return node.MakeLink(nd)
+}
+
+func (ds *HamtShard) insertChild(idx int, key string, lnk *node.Link) error {
+	if lnk == nil {
 		return os.ErrNotExist
 	}
 
 	i := ds.indexForBitPos(idx)
 	ds.bitfield.SetBit(ds.bitfield, idx, 1)
+
+	lnk.Name = ds.linkNamePrefix(idx) + key
 	sv := &shardValue{
 		key: key,
-		val: val,
+		val: lnk,
 	}
 
 	ds.children = append(ds.children[:i], append([]child{sv}, ds.children[i:]...)...)
@@ -370,11 +401,7 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*node.Link, error) {
 
 func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*node.Link) error) error {
 	return ds.walkTrie(ctx, func(sv *shardValue) error {
-		lnk, err := node.MakeLink(sv.val)
-		if err != nil {
-			return err
-		}
-
+		lnk := sv.val
 		lnk.Name = sv.key
 
 		return f(lnk)
@@ -414,7 +441,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
 	return nil
 }
 
-func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val node.Node) error {
+func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *node.Link) error {
 	idx := hv.Next(ds.tableSizeLg2)
 
 	if ds.bitfield.Bit(idx) != 1 {

unixfs/io/dirbuilder.go

@@ -48,10 +48,12 @@ func NewDirectory(dserv mdag.DAGService) *Directory {
 	return db
 }
 
+var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
+
 func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, error) {
 	pbnd, ok := nd.(*mdag.ProtoNode)
 	if !ok {
-		return nil, mdag.ErrNotProtobuf
+		return nil, ErrNotADir
 	}
 
 	pbd, err := format.FromBytes(pbnd.Data())
@@ -76,7 +78,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro
 			shard: shard,
 		}, nil
 	default:
-		return nil, fmt.Errorf("merkledag node was not a directory or shard")
+		return nil, ErrNotADir
 	}
 }