From eb7da51203ea9bbb6fe0cce8997c9e96d2b99821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Sep 2017 14:30:06 +0200 Subject: [PATCH 1/3] dag: Support multiple files in put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 113 +++++++++++++++++++++++-------------- test/sharness/t0053-dag.sh | 17 +++++- 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index c2b2b2358..1d2653831 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -8,12 +8,15 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + files "github.com/ipfs/go-ipfs/commands/files" coredag "github.com/ipfs/go-ipfs/core/coredag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" + "reflect" ) var DagCmd = &cmds.Command{ @@ -53,7 +56,7 @@ into an object of the specified format. `, }, Arguments: []cmds.Argument{ - cmds.FileArg("object data", true, false, "The object to put").EnableStdin(), + cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption("format", "f", "Format that the object will be added as.").Default("cbor"), @@ -68,12 +71,6 @@ into an object of the specified format. return } - fi, err := req.Files().NextFile() - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - ienc, _, _ := req.Option("input-enc").String() format, _, _ := req.Option("format").String() hash, _, err := req.Option("hash").String() @@ -100,52 +97,86 @@ into an object of the specified format. defer n.Blockstore.PinLock().Unlock() } - nds, err := coredag.ParseInputs(ienc, format, fi, mhType, -1) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - if len(nds) == 0 { - res.SetError(fmt.Errorf("no node returned from ParseInputs"), cmds.ErrNormal) - return + outChan := make(chan interface{}, 8) + res.SetOutput((<-chan interface{})(outChan)) + + addAllAndPin := func(f files.File) error { + for { + file, err := f.NextFile() + if err == io.EOF { + // Finished the list of files. + break + } else if err != nil { + return err + } + + nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1) + if err != nil { + return err + } + if len(nds) == 0 { + return fmt.Errorf("no node returned from ParseInputs") + } + + b := n.DAG.Batch() + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { + return err + } + } + + if err := b.Commit(); err != nil { + return err + } + + root := nds[0].Cid() + if dopin { + n.Pinning.PinWithMode(root, pin.Recursive) + + err := n.Pinning.Flush() + if err != nil { + return err + } + } + + outChan <- &OutputObject{Cid: root} + } + + return nil } - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { + go func() { + defer close(outChan) + if err := addAllAndPin(req.Files()); err != nil { res.SetError(err, cmds.ErrNormal) return } - } - - if err := b.Commit(); err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - - root := nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(root, pin.Recursive) - - err := n.Pinning.Flush() - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - } - - res.SetOutput(&OutputObject{Cid: root}) + }() }, Type: OutputObject{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - oobj, ok := res.Output().(*OutputObject) + outChan, ok := res.Output().(<-chan interface{}) if !ok { - return nil, fmt.Errorf("expected a different object in marshaler") + fmt.Println(reflect.TypeOf(res.Output())) + return nil, u.ErrCast() } - return strings.NewReader(oobj.Cid.String()), nil + marshal := func(v interface{}) (io.Reader, error) { + obj, ok := v.(*OutputObject) + if !ok { + return nil, u.ErrCast() + } + + return strings.NewReader(obj.Cid.String() + "\n"), nil + } + + return &cmds.ChannelMarshaler{ + Channel: outChan, + Marshaler: marshal, + Res: res, + }, nil }, }, } diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 5df5cc418..1e9093427 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -153,7 +153,7 @@ test_dag_cmd() { ' test_expect_success "dag put with dag-pb works output looks good" ' - printf $HASH > dag_put_exp && + echo $HASH > dag_put_exp && test_cmp dag_put_exp dag_put_out ' @@ -163,7 +163,20 @@ test_dag_cmd() { ' test_expect_success "dag put with dag-pb works output looks good" ' - printf $HASH > dag_put_exp && + echo $HASH > dag_put_exp && + test_cmp dag_put_exp dag_put_out + ' + + test_expect_success "dag put multiple files" ' + printf {\"foo\":\"bar\"} > a.json && + printf {\"foo\":\"baz\"} > b.json && + ipfs dag put a.json b.json > dag_put_out + ' + + test_expect_success "dag put multiple files output looks good" ' + echo zdpuAoKMEvka7gKGSjF9B3of1F5gE5MyMMywxTC13wCmouQrf > dag_put_exp && + echo zdpuAogmDEvpvGjMFsNTGDEU1JMYe6v69oxR8nG81EurmGHMj >> dag_put_exp && + test_cmp dag_put_exp dag_put_out ' From 03cf65bfb98dcf6a06197553686365c2d0552be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Sep 2017 20:45:14 +0200 Subject: [PATCH 2/3] dag: fix pin lock usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 1d2653831..a6e57530d 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -16,7 +16,6 @@ import ( cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" - "reflect" ) var DagCmd = &cmds.Command{ @@ -93,10 +92,6 @@ into an object of the specified format. } } - if dopin { - defer n.Blockstore.PinLock().Unlock() - } - outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) @@ -118,29 +113,37 @@ into an object of the specified format. return fmt.Errorf("no node returned from ParseInputs") } - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { + var cid *cid.Cid + err = func() error { + if dopin { + defer n.Blockstore.PinLock().Unlock() + } + + b := n.DAG.Batch() + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { + return err + } + } + + if err := b.Commit(); err != nil { return err } - } - if err := b.Commit(); err != nil { - return err - } + cid = nds[0].Cid() + if dopin { + n.Pinning.PinWithMode(cid, pin.Recursive) - root := nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(root, pin.Recursive) - - err := n.Pinning.Flush() - if err != nil { - return err + err := n.Pinning.Flush() + if err != nil { + return err + } } - } + return nil + }() - outChan <- &OutputObject{Cid: root} + outChan <- &OutputObject{Cid: cid} } return nil @@ -159,7 +162,6 @@ into an object of the specified format. cmds.Text: func(res cmds.Response) (io.Reader, error) { outChan, ok := res.Output().(<-chan interface{}) if !ok { - fmt.Println(reflect.TypeOf(res.Output())) return nil, u.ErrCast() } From f5cbc4c896c5f48b7cd9a3555471e3adcd390371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Sep 2017 23:22:50 +0200 Subject: [PATCH 3/3] dag: batch multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a6e57530d..e7f8b9176 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -96,6 +96,9 @@ into an object of the specified format. res.SetOutput((<-chan interface{})(outChan)) addAllAndPin := func(f files.File) error { + cids := cid.NewSet() + b := n.DAG.Batch() + for { file, err := f.NextFile() if err == io.EOF { @@ -113,39 +116,36 @@ into an object of the specified format. return fmt.Errorf("no node returned from ParseInputs") } - var cid *cid.Cid - err = func() error { - if dopin { - defer n.Blockstore.PinLock().Unlock() - } - - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { - return err - } - } - - if err := b.Commit(); err != nil { + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { return err } + } - cid = nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(cid, pin.Recursive) - - err := n.Pinning.Flush() - if err != nil { - return err - } - } - return nil - }() - + cid := nds[0].Cid() + cids.Add(cid) outChan <- &OutputObject{Cid: cid} } + if err := b.Commit(); err != nil { + return err + } + + if dopin { + defer n.Blockstore.PinLock().Unlock() + + cids.ForEach(func(c *cid.Cid) error { + n.Pinning.PinWithMode(c, pin.Recursive) + return nil + }) + + err := n.Pinning.Flush() + if err != nil { + return err + } + } + return nil }