From b7eae36f90f968477ecf1f76172392647db0077f Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 3 Mar 2017 17:18:57 -0500 Subject: [PATCH 1/7] adder: fix bug where errors from calls to NewAdder caused daemon to hang License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 8c88028be..ac655d621 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -202,15 +202,15 @@ You can now refer to the added file in a gateway, like so: bserv := blockservice.New(addblockstore, exch) dserv := dag.NewDAGService(bserv) - outChan := make(chan interface{}, 8) - res.SetOutput((<-chan interface{})(outChan)) - fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv) if err != nil { res.SetError(err, cmds.ErrNormal) return } + outChan := make(chan interface{}, 8) + res.SetOutput((<-chan interface{})(outChan)) + fileAdder.Out = outChan fileAdder.Chunker = chunker fileAdder.Progress = progress From cf562403d2fc0672f83bd47ed85dc41db2814b66 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sun, 5 Mar 2017 22:58:18 -0500 Subject: [PATCH 2/7] adder: make chan size a constant License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/commands/add.go b/core/commands/add.go index ac655d621..2198dbee4 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -40,6 +40,8 @@ const ( fstoreCacheOptionName = "fscache" ) +const adderOutChanSize = 8 + var AddCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add a file or directory to ipfs.", @@ -208,7 +210,7 @@ You can now refer to the added file in a gateway, like so: return } - outChan := make(chan interface{}, 8) + outChan := make(chan interface{}, adderOutChanSize) res.SetOutput((<-chan interface{})(outChan)) fileAdder.Out = outChan From 9b58fa773ad4487069bb40021264ff32738d8a0a Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 24 Mar 2017 16:18:20 -0400 Subject: [PATCH 3/7] merkledag: provide better diagnostics when Prefix.Sum fails License: MIT Signed-off-by: Kevin Atkinson --- merkledag/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/merkledag/node.go b/merkledag/node.go index 0b11b928e..ce075726d 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -266,6 +266,7 @@ func (n *ProtoNode) Cid() *cid.Cid { c, err := n.Prefix.Sum(n.RawData()) if err != nil { // programmer error + err = fmt.Errorf("invalid CID of length %d: %x: %v", len(n.RawData()), n.RawData(), err) panic(err) } From 518adec022db6ec1273dcf9c3d2cf082eabf2206 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 2 Mar 2017 21:35:04 -0500 Subject: [PATCH 4/7] adder: add support for using CidV1 License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 14 +++++++ core/coreunix/add.go | 66 ++++++++++++++++++++++-------- importer/balanced/builder.go | 6 +-- importer/helpers/dagbuilder.go | 29 ++++++++++++- importer/helpers/helpers.go | 21 +++------- importer/trickle/trickledag.go | 12 +++--- merkledag/coding.go | 2 +- merkledag/node.go | 31 +++++++++++++- mfs/dir.go | 5 +++ mfs/ops.go | 2 + mfs/system.go | 3 ++ test/sharness/t0040-add-and-cat.sh | 41 ++++++++++++++++--- test/sharness/t0043-add-w.sh | 16 ++++++++ test/sharness/t0044-add-symlink.sh | 11 +++++ unixfs/io/dirbuilder.go | 12 ++++++ 15 files changed, 220 insertions(+), 51 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 2198dbee4..1d9ce9282 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -38,6 +38,7 @@ const ( rawLeavesOptionName = "raw-leaves" noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" + cidVersionOptionName = "cid-version" ) const adderOutChanSize = 8 @@ -88,6 +89,7 @@ You can now refer to the added file in a gateway, like so: cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), + cmds.IntOption(cidVersionOptionName, "Cid version. Non-zero value will change default of 'raw-leaves' to true. (experimental)").Default(0), }, PreRun: func(req cmds.Request) error { quiet, _, _ := req.Option(quietOptionName).Bool() @@ -161,6 +163,7 @@ You can now refer to the added file in a gateway, like so: rawblks, rbset, _ := req.Option(rawLeavesOptionName).Bool() nocopy, _, _ := req.Option(noCopyOptionName).Bool() fscache, _, _ := req.Option(fstoreCacheOptionName).Bool() + cidVer, _, _ := req.Option(cidVersionOptionName).Int() if nocopy && !cfg.Experimental.FilestoreEnabled { res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"), @@ -177,6 +180,16 @@ You can now refer to the added file in a gateway, like so: return } + if cidVer >= 1 && !rbset { + rawblks = true + } + + prefix, err := dag.PrefixForCidVersion(cidVer) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + if hash { nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ //TODO: need this to be true or all files @@ -223,6 +236,7 @@ You can now refer to the added file in a gateway, like so: fileAdder.Silent = silent fileAdder.RawLeaves = rawblks fileAdder.NoCopy = nocopy + fileAdder.Prefix = prefix if hash { md := dagtest.Mock() diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 94e795c28..926030363 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -69,13 +69,7 @@ type AddedObject struct { } func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) { - mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil) - if err != nil { - return nil, err - } - return &Adder{ - mr: mr, ctx: ctx, pinning: p, blockstore: bs, @@ -87,7 +81,6 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag. Wrap: false, Chunker: "", }, nil - } // Adder holds the switches passed to the `add` command. @@ -107,13 +100,28 @@ type Adder struct { NoCopy bool Chunker string root node.Node - mr *mfs.Root + mroot *mfs.Root unlocker bs.Unlocker tempRoot *cid.Cid + Prefix cid.Prefix +} + +func (adder *Adder) mfsRoot() (*mfs.Root, error) { + if adder.mroot != nil { + return adder.mroot, nil + } + rnode := unixfs.EmptyDirNode() + rnode.SetPrefix(adder.Prefix) + mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil) + if err != nil { + return nil, err + } + adder.mroot = mr + return adder.mroot, nil } func (adder *Adder) SetMfsRoot(r *mfs.Root) { - adder.mr = r + adder.mroot = r } // Constructs a node from reader's data, and adds it. Doesn't pin. @@ -122,11 +130,13 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) { if err != nil { return nil, err } + params := ihelper.DagBuilderParams{ Dagserv: adder.dagService, RawLeaves: adder.RawLeaves, Maxlinks: ihelper.DefaultLinksPerBlock, NoCopy: adder.NoCopy, + Prefix: adder.Prefix, } if adder.Trickle { @@ -142,7 +152,11 @@ func (adder *Adder) RootNode() (node.Node, error) { return adder.root, nil } - root, err := adder.mr.GetValue().GetNode() + mr, err := adder.mfsRoot() + if err != nil { + return nil, err + } + root, err := mr.GetValue().GetNode() if err != nil { return nil, err } @@ -188,9 +202,13 @@ func (adder *Adder) PinRoot() error { } func (adder *Adder) Finalize() (node.Node, error) { - root := adder.mr.GetValue() + mr, err := adder.mfsRoot() + if err != nil { + return nil, err + } + root := mr.GetValue() - err := root.Flush() + err = root.Flush() if err != nil { return nil, err } @@ -203,7 +221,12 @@ func (adder *Adder) Finalize() (node.Node, error) { } name = children[0] - dir, ok := adder.mr.GetValue().(*mfs.Directory) + mr, err := adder.mfsRoot() + if err != nil { + return nil, err + } + + dir, ok := mr.GetValue().(*mfs.Directory) if !ok { return nil, fmt.Errorf("root is not a directory") } @@ -219,7 +242,7 @@ func (adder *Adder) Finalize() (node.Node, error) { return nil, err } - err = adder.mr.Close() + err = mr.Close() if err != nil { return nil, err } @@ -357,14 +380,18 @@ func (adder *Adder) addNode(node node.Node, path string) error { node = pi.Node } + mr, err := adder.mfsRoot() + if err != nil { + return err + } dir := gopath.Dir(path) if dir != "." { - if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil { + if err := mfs.Mkdir(mr, dir, true, false); err != nil { return err } } - if err := mfs.PutNode(adder.mr, path, node); err != nil { + if err := mfs.PutNode(mr, path, node); err != nil { return err } @@ -406,6 +433,7 @@ func (adder *Adder) addFile(file files.File) error { } dagnode := dag.NodeWithData(sdata) + dagnode.SetPrefix(adder.Prefix) _, err = adder.dagService.Add(dagnode) if err != nil { return err @@ -439,7 +467,11 @@ func (adder *Adder) addFile(file files.File) error { func (adder *Adder) addDir(dir files.File) error { log.Infof("adding directory: %s", dir.FileName()) - err := mfs.Mkdir(adder.mr, dir.FileName(), true, false) + mr, err := adder.mfsRoot() + if err != nil { + return err + } + err = mfs.Mkdir(mr, dir.FileName(), true, false) if err != nil { return err } diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index adb0ed9bb..460e3512b 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -13,7 +13,7 @@ func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) { var root *h.UnixfsNode for level := 0; !db.Done(); level++ { - nroot := h.NewUnixfsNode() + nroot := db.NewUnixfsNode() db.SetPosInfo(nroot, 0) // add our old root as a child of the new root. @@ -33,7 +33,7 @@ func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) { } if root == nil { - root = h.NewUnixfsNode() + root = db.NewUnixfsNode() } out, err := db.Add(root) @@ -72,7 +72,7 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int, offset u // while we have room AND we're not done for node.NumChildren() < db.Maxlinks() && !db.Done() { - child := h.NewUnixfsNode() + child := db.NewUnixfsNode() db.SetPosInfo(child, offset) err := fillNodeRec(db, child, depth-1, offset) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index e99937708..7ae2bae10 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -7,7 +7,9 @@ import ( "github.com/ipfs/go-ipfs/commands/files" "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" + ft "github.com/ipfs/go-ipfs/unixfs" + cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" ) @@ -23,6 +25,7 @@ type DagBuilderHelper struct { batch *dag.Batch fullPath string stat os.FileInfo + prefix cid.Prefix } type DagBuilderParams struct { @@ -33,6 +36,9 @@ type DagBuilderParams struct { // instead of using the unixfs TRaw type RawLeaves bool + // CID Prefix to use + Prefix cid.Prefix + // DAGService to write blocks to (required) Dagserv dag.DAGService @@ -48,6 +54,7 @@ func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper { dserv: dbp.Dagserv, spl: spl, rawLeaves: dbp.RawLeaves, + prefix: dbp.Prefix, maxlinks: dbp.Maxlinks, batch: dbp.Dagserv.Batch(), } @@ -103,6 +110,26 @@ func (db *DagBuilderHelper) GetDagServ() dag.DAGService { return db.dserv } +// NewUnixfsNode creates a new Unixfs node to represent a file. +func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { + n := &UnixfsNode{ + node: new(dag.ProtoNode), + ufmt: &ft.FSNode{Type: ft.TFile}, + } + n.SetPrefix(db.prefix) + return n +} + +// NewUnixfsBlock creates a new Unixfs node to represent a raw data block +func (db *DagBuilderHelper) NewUnixfsBlock() *UnixfsNode { + n := &UnixfsNode{ + node: new(dag.ProtoNode), + ufmt: &ft.FSNode{Type: ft.TRaw}, + } + n.SetPrefix(db.prefix) + return n +} + // FillNodeLayer will add datanodes as children to the give node until // at most db.indirSize ndoes are added // @@ -143,7 +170,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { raw: true, }, nil } else { - blk := NewUnixfsBlock() + blk := db.NewUnixfsBlock() blk.SetData(data) return blk, nil } diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index db3504a78..36c6eaf67 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -10,6 +10,7 @@ import ( pi "github.com/ipfs/go-ipfs/thirdparty/posinfo" ft "github.com/ipfs/go-ipfs/unixfs" + cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" ) @@ -48,22 +49,6 @@ type UnixfsNode struct { posInfo *pi.PosInfo } -// NewUnixfsNode creates a new Unixfs node to represent a file -func NewUnixfsNode() *UnixfsNode { - return &UnixfsNode{ - node: new(dag.ProtoNode), - ufmt: &ft.FSNode{Type: ft.TFile}, - } -} - -// NewUnixfsBlock creates a new Unixfs node to represent a raw data block -func NewUnixfsBlock() *UnixfsNode { - return &UnixfsNode{ - node: new(dag.ProtoNode), - ufmt: &ft.FSNode{Type: ft.TRaw}, - } -} - // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { mb, err := ft.FSNodeFromBytes(nd.Data()) @@ -77,6 +62,10 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { }, nil } +func (n *UnixfsNode) SetPrefix(prefix cid.Prefix) { + n.node.SetPrefix(prefix) +} + func (n *UnixfsNode) NumChildren() int { return n.ufmt.NumChildren() } diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 9b773a1f1..257cbd718 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -18,13 +18,13 @@ import ( const layerRepeat = 4 func TrickleLayout(db *h.DagBuilderHelper) (node.Node, error) { - root := h.NewUnixfsNode() + root := db.NewUnixfsNode() if err := db.FillNodeLayer(root); err != nil { return nil, err } for level := 1; !db.Done(); level++ { for i := 0; i < layerRepeat && !db.Done(); i++ { - next := h.NewUnixfsNode() + next := db.NewUnixfsNode() if err := fillTrickleRec(db, next, level); err != nil { return nil, err } @@ -54,7 +54,7 @@ func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int) error for i := 1; i < depth && !db.Done(); i++ { for j := 0; j < layerRepeat && !db.Done(); j++ { - next := h.NewUnixfsNode() + next := db.NewUnixfsNode() if err := fillTrickleRec(db, next, i); err != nil { return err } @@ -117,7 +117,7 @@ func TrickleAppend(ctx context.Context, basen node.Node, db *h.DagBuilderHelper) // Now, continue filling out tree like normal for i := n; !db.Done(); i++ { for j := 0; j < layerRepeat && !db.Done(); j++ { - next := h.NewUnixfsNode() + next := db.NewUnixfsNode() err := fillTrickleRec(db, next, i) if err != nil { return nil, err @@ -162,7 +162,7 @@ func appendFillLastChild(ctx context.Context, ufsn *h.UnixfsNode, depth int, lay // Partially filled depth layer if layerFill != 0 { for ; layerFill < layerRepeat && !db.Done(); layerFill++ { - next := h.NewUnixfsNode() + next := db.NewUnixfsNode() err := fillTrickleRec(db, next, depth) if err != nil { return err @@ -211,7 +211,7 @@ func trickleAppendRec(ctx context.Context, ufsn *h.UnixfsNode, db *h.DagBuilderH // Now, continue filling out tree like normal for i := n; i < depth && !db.Done(); i++ { for j := 0; j < layerRepeat && !db.Done(); j++ { - next := h.NewUnixfsNode() + next := db.NewUnixfsNode() if err := fillTrickleRec(db, next, i); err != nil { return nil, err } diff --git a/merkledag/coding.go b/merkledag/coding.go index 25f43e0d6..b5c2075da 100644 --- a/merkledag/coding.go +++ b/merkledag/coding.go @@ -86,7 +86,7 @@ func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { if n.cached == nil { if n.Prefix.Codec == 0 { // unset - n.Prefix = defaultCidPrefix + n.Prefix = v0CidPrefix } c, err := n.Prefix.Sum(n.encoded) if err != nil { diff --git a/merkledag/node.go b/merkledag/node.go index ce075726d..f4c4a0839 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -28,13 +28,37 @@ type ProtoNode struct { Prefix cid.Prefix } -var defaultCidPrefix = cid.Prefix{ +var v0CidPrefix = cid.Prefix{ Codec: cid.DagProtobuf, MhLength: -1, MhType: mh.SHA2_256, Version: 0, } +var v1CidPrefix = cid.Prefix{ + Codec: cid.DagProtobuf, + MhLength: -1, + MhType: mh.SHA2_256, + Version: 1, +} + +func PrefixForCidVersion(version int) (cid.Prefix, error) { + switch version { + case 0: + return v0CidPrefix, nil + case 1: + return v1CidPrefix, nil + default: + return cid.Prefix{}, fmt.Errorf("unknown CID version: %d", version) + } +} + +func (n *ProtoNode) SetPrefix(prefix cid.Prefix) { + n.Prefix = prefix + n.encoded = nil + n.cached = nil +} + type LinkSlice []*node.Link func (ls LinkSlice) Len() int { return len(ls) } @@ -158,6 +182,9 @@ func (n *ProtoNode) Copy() node.Node { nnode.links = make([]*node.Link, len(n.links)) copy(nnode.links, n.links) } + + nnode.Prefix = n.Prefix + return nnode } @@ -260,7 +287,7 @@ func (n *ProtoNode) Cid() *cid.Cid { } if n.Prefix.Codec == 0 { - n.Prefix = defaultCidPrefix + n.Prefix = v0CidPrefix } c, err := n.Prefix.Sum(n.RawData()) diff --git a/mfs/dir.go b/mfs/dir.go index a0a9205b6..11280bc17 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -15,6 +15,7 @@ import ( uio "github.com/ipfs/go-ipfs/unixfs/io" ufspb "github.com/ipfs/go-ipfs/unixfs/pb" + cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" ) @@ -57,6 +58,10 @@ func NewDirectory(ctx context.Context, name string, node node.Node, parent child }, nil } +func (d *Directory) SetPrefix(prefix cid.Prefix) { + d.dirbuilder.SetPrefix(prefix) +} + // closeChild updates the child by the given name to the dag node 'nd' // and changes its own dag node func (d *Directory) closeChild(name string, nd node.Node, sync bool) error { diff --git a/mfs/ops.go b/mfs/ops.go index 694a001b0..f84540a6a 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -134,6 +134,7 @@ func Mkdir(r *Root, pth string, mkparents bool, flush bool) error { if err != nil { return err } + mkd.SetPrefix(r.Prefix) fsn = mkd } else if err != nil { return err @@ -152,6 +153,7 @@ func Mkdir(r *Root, pth string, mkparents bool, flush bool) error { return err } } + final.SetPrefix(r.Prefix) if flush { err := final.Flush() diff --git a/mfs/system.go b/mfs/system.go index 8bc6893c8..a28a7fb10 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -61,6 +61,9 @@ type Root struct { dserv dag.DAGService Type string + + // Prefix to use for any children created + Prefix cid.Prefix } type PubFunc func(context.Context, *cid.Cid) error diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index d2be8b399..4b2d636a4 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -100,7 +100,7 @@ test_add_cat_5MB() { test_cmp sha1_expected sha1_actual ' - test_expect_success "'ipfs add bigfile' succeeds" ' + test_expect_success "'ipfs add $ADD_FLAGS bigfile' succeeds" ' ipfs add $ADD_FLAGS mountdir/bigfile >actual || test_fsh cat daemon_err ' @@ -142,6 +142,9 @@ test_add_cat_raw() { } test_add_cat_expensive() { + ADD_FLAGS="$1" + HASH="$2" + test_expect_success EXPENSIVE "generate 100MB file using go-random" ' random 104857600 42 >mountdir/bigfile ' @@ -152,12 +155,11 @@ test_add_cat_expensive() { test_cmp sha1_expected sha1_actual ' - test_expect_success EXPENSIVE "ipfs add bigfile succeeds" ' - ipfs add mountdir/bigfile >actual + test_expect_success EXPENSIVE "ipfs add $ADD_FLAGS bigfile succeeds" ' + ipfs add $ADD_FLAGS mountdir/bigfile >actual ' test_expect_success EXPENSIVE "ipfs add bigfile output looks good" ' - HASH="QmU9SWAPPmNEKZB8umYMmjYvN7VyHqABNvdA6GUi4MMEz3" && echo "added $HASH bigfile" >expected && test_cmp expected actual ' @@ -391,6 +393,16 @@ MARS="zb2rhZdTkQNawVajsTNiYc9cTPHqgLdJVvBRkZok9RjkgQYRU" VENUS="zb2rhn6TGvnUaMAg4VV4y9HVx5W42HihcH4jsyrDv8mkepFqq" add_directory '--raw-leaves' +PLANETS="zdj7Wnbun6P41Z5ddTkNvZaDTmQ8ZLdiKFcJrL9sV87rPScMP" +MARS="zb2rhZdTkQNawVajsTNiYc9cTPHqgLdJVvBRkZok9RjkgQYRU" +VENUS="zb2rhn6TGvnUaMAg4VV4y9HVx5W42HihcH4jsyrDv8mkepFqq" +add_directory '--cid-version=1' + +PLANETS="zdj7WiC51v78BjBcmZR7uuBvmDWxSn5EDr5MiyTwE18e8qvb7" +MARS="zdj7WWx6fGNrNGkdpkuTAxCjKbQ1pPtarqA6VQhedhLTZu34J" +VENUS="zdj7WbB1BUF8WejmVpQCmMLd1RbPnxJtvAj1Lep6eTmXRFbrz" +add_directory '--cid-version=1 --raw-leaves=false' + test_expect_success "'ipfs add -rn' succeeds" ' mkdir -p mountdir/moons/jupiter && @@ -425,7 +437,20 @@ test_add_cat_5MB "" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" test_add_cat_5MB --raw-leaves "QmbdLHCmdi48eM8T7D67oXjA1S2Puo8eMfngdHhdPukFd6" -test_add_cat_expensive +# note: the specified hash implies that internal nodes are stored +# using CidV1 and leaves are stored using raw blocks +test_add_cat_5MB --cid-version=1 "zdj7WiiaedqVBXjX4SNqx3jfuZideDqdLYnDzCDJ66JDMK9o2" + +# note: the specified hash implies that internal nodes are stored +# using CidV1 and leaves are stored using CidV1 but using the legacy +# format (i.e. not raw) +test_add_cat_5MB '--cid-version=1 --raw-leaves=false' "zdj7WfgEsj897BBZj2mcfsRLhaPZcCixPV2G7DkWgF1Wdr64P" + +test_add_cat_expensive "" "QmU9SWAPPmNEKZB8umYMmjYvN7VyHqABNvdA6GUi4MMEz3" + +# note: the specified hash implies that internal nodes are stored +# using CidV1 and leaves are stored using raw blocks +test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYhqxeJyW1wgCq" test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=true&r=true&stream-channels=true:" @@ -433,6 +458,12 @@ test_add_pwd_is_symlink test_add_cat_raw +test_expect_success "ipfs add --cid-version=9 fails" ' + echo "context" > afile.txt && + test_must_fail ipfs add --cid-version=9 afile.txt 2>&1 | tee add_out && + grep -q "unknown CID version" add_out +' + test_kill_ipfs_daemon # should work offline diff --git a/test/sharness/t0043-add-w.sh b/test/sharness/t0043-add-w.sh index 40e9649b7..cd4b5d1e5 100755 --- a/test/sharness/t0043-add-w.sh +++ b/test/sharness/t0043-add-w.sh @@ -27,6 +27,14 @@ added QmYC3u5jGWuyFwvTxtvLYm2K3SpWZ31tg3NjpVVvh9cJaJ _jo7/wzvsihy added QmQkib3f9XNX5sj6WEahLUPFpheTcwSRJwUCSvjcv8b9by _jo7 added QmNQoesMj1qp8ApE51NbtTjFYksyzkezPD4cat7V2kzbKN ' +add_w_d1_v1='added zb2rhjXyHbbgwgtAUwHtpBd8iXLgK22ZjVmaiJSMNmqBTpXS3 _jo7/-s782qgs +added zb2rhi6PQqQFbS4QsvrV8sL9ue1fvFoqtLVqogNPCZri8rquN _jo7/15totauzkak- +added zb2rhjQthC6LgnNZztpsF9LcfPxznh3cJnmzUx8dnSqNqJ8Yz _jo7/galecuirrj4r +added zb2rhYh9hgDw1DpaZfLUU5MkKNezPWjPTkgGQPiTyLpZYu3jn _jo7/mzo50r-1xidf5zx +added zb2rhZK5xwEUhY4uskfj4sn979aCH27cnqseVVznYDn7NFWtt _jo7/wzvsihy +added zdj7WfNC8EZchqskczxsgrVEqwLVpksQ9B5kopf391jVbCGwv _jo7 +added zdj7Wn5jf686mfYE8gUKWzY7aTjp5eAQcecD8q4ZtqLJbDNxe ' + add_w_d2='added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 added QmU9Jqks8TPu4vFr6t7EKkAKQrSJuEujNj1AkzoCeTEDFJ gnz66h/1k0xpx34 added QmSLYZycXAufRw3ePMVH2brbtYWCcWsmksGLbHcT8ia9Ke gnz66h/9cwudvacx @@ -133,6 +141,14 @@ test_add_w() { test_sort_cmp expected actual ' + test_expect_success "ipfs add -w -r (dir) --cid-version=1 succeeds" ' + ipfs add -r -w --cid-version=1 m/t_1wp-8a2/_jo7 >actual + ' + + test_expect_success "ipfs add -w -r (dir) --cid-version=1 is correct" ' + echo "$add_w_d1_v1" >expected && + test_sort_cmp expected actual + ' } test_init_ipfs diff --git a/test/sharness/t0044-add-symlink.sh b/test/sharness/t0044-add-symlink.sh index e2f5f4164..e7ac10635 100755 --- a/test/sharness/t0044-add-symlink.sh +++ b/test/sharness/t0044-add-symlink.sh @@ -27,6 +27,17 @@ test_add_symlinks() { test_cmp filehash_exp filehash_out ' + test_expect_success "ipfs add --cid-version=1 files succeeds" ' + ipfs add -q -r --cid-version=1 files >filehash_all && + tail -n 1 filehash_all >filehash_out + ' + + test_expect_success "output looks good" ' + # note this hash implies all internal nodes are stored using CidV1 + echo zdj7WZDQ2xMmr4qn6aRZTsE9fCUs2KmvPigpHdpssqUobwcWK > filehash_exp && + test_cmp filehash_exp filehash_out + ' + test_expect_success "adding a symlink adds the link itself" ' ipfs add -q files/bar/baz > goodlink_out ' diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index c0587480d..a8663763c 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -8,6 +8,7 @@ import ( mdag "github.com/ipfs/go-ipfs/merkledag" format "github.com/ipfs/go-ipfs/unixfs" hamt "github.com/ipfs/go-ipfs/unixfs/hamt" + cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" ) @@ -79,6 +80,17 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro } } +// SetPrefix sets the prefix of the root node +func (d *Directory) SetPrefix(prefix cid.Prefix) { + if d.dirnode != nil { + d.dirnode.SetPrefix(prefix) + } + // FIXME: Should we do this? -- kevina + //if d.shard != nil { + // d.shard.SetPrefix(prefix) + //} +} + // AddChild adds a (name, key)-pair to the root node. func (d *Directory) AddChild(ctx context.Context, name string, nd node.Node) error { if d.shard == nil { From 076d0bd9b1a0102ea889e3381a20287009825777 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 24 Mar 2017 16:41:50 -0400 Subject: [PATCH 5/7] merkeldag: change SetPrefix param to a pointer and reset the prefix on nil License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 2 +- core/coreunix/add.go | 3 ++- importer/helpers/dagbuilder.go | 6 +++--- importer/helpers/helpers.go | 2 +- merkledag/node.go | 17 ++++++++++++----- mfs/dir.go | 2 +- mfs/system.go | 2 +- unixfs/io/dirbuilder.go | 2 +- 8 files changed, 22 insertions(+), 14 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 1d9ce9282..a16817db9 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -236,7 +236,7 @@ You can now refer to the added file in a gateway, like so: fileAdder.Silent = silent fileAdder.RawLeaves = rawblks fileAdder.NoCopy = nocopy - fileAdder.Prefix = prefix + fileAdder.Prefix = &prefix if hash { md := dagtest.Mock() diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 926030363..e8bf3a224 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -103,7 +103,7 @@ type Adder struct { mroot *mfs.Root unlocker bs.Unlocker tempRoot *cid.Cid - Prefix cid.Prefix + Prefix *cid.Prefix } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -113,6 +113,7 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) { rnode := unixfs.EmptyDirNode() rnode.SetPrefix(adder.Prefix) mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil) + mr.Prefix = adder.Prefix if err != nil { return nil, err } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 7ae2bae10..a06b38c59 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -25,7 +25,7 @@ type DagBuilderHelper struct { batch *dag.Batch fullPath string stat os.FileInfo - prefix cid.Prefix + prefix *cid.Prefix } type DagBuilderParams struct { @@ -36,8 +36,8 @@ type DagBuilderParams struct { // instead of using the unixfs TRaw type RawLeaves bool - // CID Prefix to use - Prefix cid.Prefix + // CID Prefix to use if set + Prefix *cid.Prefix // DAGService to write blocks to (required) Dagserv dag.DAGService diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 36c6eaf67..bc6d92e2f 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -62,7 +62,7 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { }, nil } -func (n *UnixfsNode) SetPrefix(prefix cid.Prefix) { +func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) { n.node.SetPrefix(prefix) } diff --git a/merkledag/node.go b/merkledag/node.go index f4c4a0839..4090a99b8 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -53,10 +53,17 @@ func PrefixForCidVersion(version int) (cid.Prefix, error) { } } -func (n *ProtoNode) SetPrefix(prefix cid.Prefix) { - n.Prefix = prefix - n.encoded = nil - n.cached = nil +// SetPrefix sets the prefix if it is non nil, if prefix is nil then +// it resets it the default value +func (n *ProtoNode) SetPrefix(prefix *cid.Prefix) { + if prefix == nil { + n.Prefix = v0CidPrefix + } else { + n.Prefix = *prefix + n.Prefix.Codec = cid.DagProtobuf + n.encoded = nil + n.cached = nil + } } type LinkSlice []*node.Link @@ -287,7 +294,7 @@ func (n *ProtoNode) Cid() *cid.Cid { } if n.Prefix.Codec == 0 { - n.Prefix = v0CidPrefix + n.SetPrefix(nil) } c, err := n.Prefix.Sum(n.RawData()) diff --git a/mfs/dir.go b/mfs/dir.go index 11280bc17..102ee15cb 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -58,7 +58,7 @@ func NewDirectory(ctx context.Context, name string, node node.Node, parent child }, nil } -func (d *Directory) SetPrefix(prefix cid.Prefix) { +func (d *Directory) SetPrefix(prefix *cid.Prefix) { d.dirbuilder.SetPrefix(prefix) } diff --git a/mfs/system.go b/mfs/system.go index a28a7fb10..4ed84d83b 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -63,7 +63,7 @@ type Root struct { Type string // Prefix to use for any children created - Prefix cid.Prefix + Prefix *cid.Prefix } type PubFunc func(context.Context, *cid.Cid) error diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index a8663763c..a5dce7f48 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -81,7 +81,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro } // SetPrefix sets the prefix of the root node -func (d *Directory) SetPrefix(prefix cid.Prefix) { +func (d *Directory) SetPrefix(prefix *cid.Prefix) { if d.dirnode != nil { d.dirnode.SetPrefix(prefix) } From 8a47786c6727a0433c7a298b2a779f9817773695 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 31 Mar 2017 01:00:50 -0400 Subject: [PATCH 6/7] hamt: support using CIDv1 by allowing the prefix to be set License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0260-sharding-flag.sh | 19 +++++++++++++++++++ unixfs/hamt/hamt.go | 7 +++++++ unixfs/io/dirbuilder.go | 7 +++---- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/test/sharness/t0260-sharding-flag.sh b/test/sharness/t0260-sharding-flag.sh index ece3fe5de..a5897cdbb 100755 --- a/test/sharness/t0260-sharding-flag.sh +++ b/test/sharness/t0260-sharding-flag.sh @@ -55,4 +55,23 @@ test_expect_success "sharded and unsharded output look the same" ' test_cmp sharded_out unsharded_out ' +test_add_large_dir_v1() { + exphash="$1" + test_expect_success "ipfs add (CIDv1) on very large directory succeeds" ' + ipfs add -r -q --cid-version=1 testdata | tail -n1 > sharddir_out && + echo "$exphash" > sharddir_exp && + test_cmp sharddir_exp sharddir_out + ' +} + +# this hash implies both the directory and the leaf entries are CIDv1 +SHARDEDV1="zdj7WX91spg4DsnNpvoBLjyjXUGgcTTWavygBbSifpmJdgPUA" +test_add_large_dir_v1 "$SHARDEDV1" + +test_launch_ipfs_daemon + +test_add_large_dir_v1 "$SHARDEDV1" + +test_kill_ipfs_daemon + test_done diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index 4c3f4f913..17aa01733 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -31,6 +31,7 @@ import ( format "github.com/ipfs/go-ipfs/unixfs" upb "github.com/ipfs/go-ipfs/unixfs/pb" + cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" "gx/ipfs/QmfJHywXQu98UeZtGJBQrPAR6AtmDjjbe3qjTo9piXHPnx/murmur3" @@ -50,6 +51,7 @@ type HamtShard struct { tableSize int tableSizeLg2 int + prefix *cid.Prefix hashFunc uint64 prefixPadStr string @@ -123,9 +125,14 @@ func NewHamtFromDag(dserv dag.DAGService, nd node.Node) (*HamtShard, error) { return ds, nil } +func (ds *HamtShard) SetPrefix(prefix *cid.Prefix) { + ds.prefix = prefix +} + // Node serializes the HAMT structure into a merkledag node with unixfs formatting func (ds *HamtShard) Node() (node.Node, error) { out := new(dag.ProtoNode) + out.SetPrefix(ds.prefix) // TODO: optimized 'for each set bit' for i := 0; i < ds.tableSize; i++ { diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index a5dce7f48..bcf9770f4 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -85,10 +85,9 @@ func (d *Directory) SetPrefix(prefix *cid.Prefix) { if d.dirnode != nil { d.dirnode.SetPrefix(prefix) } - // FIXME: Should we do this? -- kevina - //if d.shard != nil { - // d.shard.SetPrefix(prefix) - //} + if d.shard != nil { + d.shard.SetPrefix(prefix) + } } // AddChild adds a (name, key)-pair to the root node. From 9350c513f14dadb6226bee9dc81c1486f53a5b91 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 25 Apr 2017 23:47:48 -0400 Subject: [PATCH 7/7] Documentation License: MIT Signed-off-by: Kevin Atkinson --- importer/helpers/helpers.go | 1 + merkledag/node.go | 3 ++- mfs/dir.go | 1 + unixfs/hamt/hamt.go | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index bc6d92e2f..317fd60d0 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -62,6 +62,7 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { }, nil } +// SetPrefix sets the CID Prefix func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) { n.node.SetPrefix(prefix) } diff --git a/merkledag/node.go b/merkledag/node.go index 4090a99b8..fa575097a 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -42,6 +42,7 @@ var v1CidPrefix = cid.Prefix{ Version: 1, } +// PrefixForCidVersion returns the Protobuf prefix for a given CID version func PrefixForCidVersion(version int) (cid.Prefix, error) { switch version { case 0: @@ -53,7 +54,7 @@ func PrefixForCidVersion(version int) (cid.Prefix, error) { } } -// SetPrefix sets the prefix if it is non nil, if prefix is nil then +// SetPrefix sets the CID prefix if it is non nil, if prefix is nil then // it resets it the default value func (n *ProtoNode) SetPrefix(prefix *cid.Prefix) { if prefix == nil { diff --git a/mfs/dir.go b/mfs/dir.go index 102ee15cb..60cae39c7 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -58,6 +58,7 @@ func NewDirectory(ctx context.Context, name string, node node.Node, parent child }, nil } +// SetPrefix sets the CID prefix func (d *Directory) SetPrefix(prefix *cid.Prefix) { d.dirbuilder.SetPrefix(prefix) } diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index 17aa01733..ccdffe7e4 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -125,6 +125,7 @@ func NewHamtFromDag(dserv dag.DAGService, nd node.Node) (*HamtShard, error) { return ds, nil } +// SetPrefix sets the CID Prefix func (ds *HamtShard) SetPrefix(prefix *cid.Prefix) { ds.prefix = prefix }