From 9e8b6e5b50f5dc00aadc7ed9d6b9399ee609f31b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 27 Oct 2021 05:07:31 +1100 Subject: [PATCH] chore: replace go-merkledag walk with go-ipld-prime traversal for dag export (#8506) * chore: replace go-merkledag walk with go-ipld-prime traversal for dag export This is "safe" now because we can limit duplicate block loads like go-merkledag does and won't get trapped taking a long time for complex DAGs. We can do this while we're using an exhaustive selector (like ExploreAll here) but will need an alternative strategy when we go for arbitrary selectors. --- core/commands/dag/export.go | 49 +++++++++++++++---------------------- go.mod | 4 +-- go.sum | 8 +++--- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index bff9dc479..7290002e7 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -1,6 +1,7 @@ package dagcmd import ( + "context" "errors" "fmt" "io" @@ -8,17 +9,18 @@ import ( "time" "github.com/cheggaaa/pb" + blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" ipld "github.com/ipfs/go-ipld-format" - mdag "github.com/ipfs/go-merkledag" + iface "github.com/ipfs/interface-go-ipfs-core" cmds "github.com/ipfs/go-ipfs-cmds" gocar "github.com/ipld/go-car" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - c, err := cid.Decode(req.Arguments[0]) if err != nil { return fmt.Errorf( @@ -32,24 +34,6 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } - // Code disabled until descent-issue in go-ipld-prime is fixed - // https://github.com/ribasushi/gip-muddle-up - // - // sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder()) - // car := gocar.NewSelectiveCar( - // req.Context, - // , - // []gocar.Dag{gocar.Dag{ - // Root: c, - // Selector: sb.ExploreRecursive( - // gipselector.RecursionLimitNone(), - // sb.ExploreAll(sb.ExploreRecursiveEdge()), - // ).Node(), - // }}, - // ) - // ... - // if err := car.Write(pipeW); err != nil {} - pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error @@ -61,15 +45,12 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment close(errCh) }() - if err := gocar.WriteCar( - req.Context, - mdag.NewSession( - req.Context, - api.Dag(), - ), - []cid.Cid{c}, - pipeW, - ); err != nil { + store := dagStore{dag: api.Dag(), ctx: req.Context} + dag := gocar.Dag{Root: c, Selector: selectorparse.CommonSelector_ExploreAllRecursively} + // TraverseLinksOnlyOnce is safe for an exhaustive selector but won't be when we allow + // arbitrary selectors here + car := gocar.NewSelectiveCar(req.Context, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce()) + if err := car.Write(pipeW); err != nil { errCh <- err } }() @@ -153,3 +134,13 @@ func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error { } } } + +type dagStore struct { + dag iface.APIDagService + ctx context.Context +} + +func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) { + obj, err := ds.dag.Get(ds.ctx, c) + return obj, err +} diff --git a/go.mod b/go.mod index 93227d61d..739d85be3 100644 --- a/go.mod +++ b/go.mod @@ -58,9 +58,9 @@ require ( github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.5.1 github.com/ipfs/tar-utils v0.0.1 - github.com/ipld/go-car v0.3.1 + github.com/ipld/go-car v0.3.2 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.12.2 + github.com/ipld/go-ipld-prime v0.12.3 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 diff --git a/go.sum b/go.sum index 7422583a9..643d42061 100644 --- a/go.sum +++ b/go.sum @@ -571,16 +571,16 @@ github.com/ipfs/interface-go-ipfs-core v0.5.1 h1:1KMM7RkjUD8W5fSoRsa9xR6ZMzeL8fL github.com/ipfs/interface-go-ipfs-core v0.5.1/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE= github.com/ipfs/tar-utils v0.0.1 h1:8Na0KBD6GddGyXwU4rXNtVTE24iuZws8mENJQPLG7W4= github.com/ipfs/tar-utils v0.0.1/go.mod h1:ACflm9wXvV9w0eMJt6yYXxS2zuIV+yXGNwbuq1bhLeE= -github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= -github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= +github.com/ipld/go-car v0.3.2 h1:V9wt/80FNfbMRWSD98W5br6fyjUAyVgI2lDOTZX16Lg= +github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.12.2 h1:StIquYvKIRuSEAtjJDr39fyzBtziioHPwVC75tBiXzo= -github.com/ipld/go-ipld-prime v0.12.2/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= +github.com/ipld/go-ipld-prime v0.12.3 h1:furVobw7UBLQZwlEwfE26tYORy3PAK8VYSgZOSr3JMQ= +github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=