From 0597a04924c8e456ea47d7236b8296ed00c4de45 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 30 Apr 2017 13:37:37 -0700 Subject: [PATCH] Fix sharding memory growth, and fix resolver for unixfs paths License: MIT Signed-off-by: Jeromy --- core/coreunix/cat.go | 7 +++- path/resolver.go | 4 +- test/sharness/t0260-sharding-flag.sh | 5 +++ unixfs/hamt/hamt.go | 63 ++++++++++++++++++++-------- unixfs/io/dirbuilder.go | 6 ++- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/core/coreunix/cat.go b/core/coreunix/cat.go index 240307d2c..d58f190a3 100644 --- a/core/coreunix/cat.go +++ b/core/coreunix/cat.go @@ -9,7 +9,12 @@ import ( ) func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (uio.DagReader, error) { - dagNode, err := core.Resolve(ctx, n.Namesys, n.Resolver, path.Path(pstr)) + r := &path.Resolver{ + DAG: n.DAG, + ResolveOnce: uio.ResolveUnixfsOnce, + } + + dagNode, err := core.Resolve(ctx, n.Namesys, r, path.Path(pstr)) if err != nil { return nil, err } diff --git a/path/resolver.go b/path/resolver.go index 4ebde479f..84a6fe66c 100644 --- a/path/resolver.go +++ b/path/resolver.go @@ -163,7 +163,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri ctx, cancel = context.WithTimeout(ctx, time.Minute) defer cancel() - lnk, rest, err := nd.ResolveLink(names) + lnk, err := s.ResolveOnce(ctx, s.DAG, nd, names[0]) if err == dag.ErrLinkNotFound { return result, ErrNoLink{Name: names[0], Node: nd.Cid()} } else if err != nil { @@ -177,7 +177,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri nd = nextnode result = append(result, nextnode) - names = rest + names = names[1:] } return result, nil } diff --git a/test/sharness/t0260-sharding-flag.sh b/test/sharness/t0260-sharding-flag.sh index 39d96712b..7bb75ab66 100755 --- a/test/sharness/t0260-sharding-flag.sh +++ b/test/sharness/t0260-sharding-flag.sh @@ -68,6 +68,11 @@ test_add_large_dir_v1() { echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' + + test_expect_success "can access a path under the dir" ' + ipfs cat "$exphash/file20" > file20_out && + test_cmp testdata/file20 file20_out + ' } # this hash implies both the directory and the leaf entries are CIDv1 diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index ccdffe7e4..ceda529c9 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -62,7 +62,7 @@ type HamtShard struct { // child can either be another shard, or a leaf node value type child interface { - Node() (node.Node, error) + Link() (*node.Link, error) Label() string } @@ -144,12 +144,12 @@ func (ds *HamtShard) Node() (node.Node, error) { cindex := ds.indexForBitPos(i) ch := ds.children[cindex] if ch != nil { - cnd, err := ch.Node() + clnk, err := ch.Link() if err != nil { return nil, err } - err = out.AddNodeLinkClean(ds.linkNamePrefix(i)+ch.Label(), cnd) + err = out.AddRawLink(ds.linkNamePrefix(i)+ch.Label(), clnk) if err != nil { return nil, err } @@ -188,10 +188,10 @@ func (ds *HamtShard) Node() (node.Node, error) { type shardValue struct { key string - val node.Node + val *node.Link } -func (sv *shardValue) Node() (node.Node, error) { +func (sv *shardValue) Link() (*node.Link, error) { return sv.val, nil } @@ -214,7 +214,18 @@ func (ds *HamtShard) Label() string { // Set sets 'name' = nd in the HAMT func (ds *HamtShard) Set(ctx context.Context, name string, nd node.Node) error { hv := &hashBits{b: hash([]byte(name))} - return ds.modifyValue(ctx, hv, name, nd) + _, err := ds.dserv.Add(nd) + if err != nil { + return err + } + + lnk, err := node.MakeLink(nd) + if err != nil { + return err + } + lnk.Name = ds.linkNamePrefix(0) + name + + return ds.modifyValue(ctx, hv, name, lnk) } // Remove deletes the named entry if it exists, this operation is idempotent. @@ -226,13 +237,16 @@ func (ds *HamtShard) Remove(ctx context.Context, name string) error { func (ds *HamtShard) Find(ctx context.Context, name string) (node.Node, error) { hv := &hashBits{b: hash([]byte(name))} - var out node.Node + var out *node.Link err := ds.getValue(ctx, hv, name, func(sv *shardValue) error { out = sv.val return nil }) + if err != nil { + return nil, err + } - return out, err + return ds.dserv.Get(ctx, out.Cid) } // getChild returns the i'th child of this shard. If it is cached in the @@ -291,9 +305,10 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) { c = cds } else { + lnk2 := *lnk c = &shardValue{ key: lnk.Name[ds.maxpadlen:], - val: nd, + val: &lnk2, } } @@ -305,16 +320,32 @@ func (ds *HamtShard) setChild(i int, c child) { ds.children[i] = c } -func (ds *HamtShard) insertChild(idx int, key string, val node.Node) error { - if val == nil { +func (ds *HamtShard) Link() (*node.Link, error) { + nd, err := ds.Node() + if err != nil { + return nil, err + } + + _, err = ds.dserv.Add(nd) + if err != nil { + return nil, err + } + + return node.MakeLink(nd) +} + +func (ds *HamtShard) insertChild(idx int, key string, lnk *node.Link) error { + if lnk == nil { return os.ErrNotExist } i := ds.indexForBitPos(idx) ds.bitfield.SetBit(ds.bitfield, idx, 1) + + lnk.Name = ds.linkNamePrefix(idx) + key sv := &shardValue{ key: key, - val: val, + val: lnk, } ds.children = append(ds.children[:i], append([]child{sv}, ds.children[i:]...)...) @@ -370,11 +401,7 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*node.Link, error) { func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*node.Link) error) error { return ds.walkTrie(ctx, func(sv *shardValue) error { - lnk, err := node.MakeLink(sv.val) - if err != nil { - return err - } - + lnk := sv.val lnk.Name = sv.key return f(lnk) @@ -414,7 +441,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e return nil } -func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val node.Node) error { +func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *node.Link) error { idx := hv.Next(ds.tableSizeLg2) if ds.bitfield.Bit(idx) != 1 { diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index bcf9770f4..285992081 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -48,10 +48,12 @@ func NewDirectory(dserv mdag.DAGService) *Directory { return db } +var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard") + func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, error) { pbnd, ok := nd.(*mdag.ProtoNode) if !ok { - return nil, mdag.ErrNotProtobuf + return nil, ErrNotADir } pbd, err := format.FromBytes(pbnd.Data()) @@ -76,7 +78,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro shard: shard, }, nil default: - return nil, fmt.Errorf("merkledag node was not a directory or shard") + return nil, ErrNotADir } }