From 521a29956b35ceeba775d6f699edc02c29c1857e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Jul 2019 17:30:35 -0700 Subject: [PATCH 1/4] fix and improve the writable gateway 1. Fix handling of PUT. The simple implementation was the correct implementation, I have no idea what was going on here. 2. Use MFS everywhere to reduce code duplication and add support for sharded directories. 3. _Correctly_ block IPNS. 4. Remove the dependency on `core.IpfsNode`. 5. Remove support for putting empty directories with a well-known CID. It was useless as directories are automatically created. --- core/corehttp/gateway.go | 2 +- core/corehttp/gateway_handler.go | 297 +++++++++++------------ go.mod | 4 +- go.sum | 6 +- test/sharness/t0111-gateway-writeable.sh | 29 ++- 5 files changed, 163 insertions(+), 175 deletions(-) diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index d3600bb73..24d26739b 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -87,7 +87,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption { "X-Stream-Output", }, headers[ACEHeadersName]...)) - gateway := newGatewayHandler(n, GatewayConfig{ + gateway := newGatewayHandler(GatewayConfig{ Headers: headers, Writable: writable, PathPrefixes: cfg.Gateway.PathPrefixes, diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 1aa343a55..384f6dc45 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -2,30 +2,23 @@ package corehttp import ( "context" - "errors" "fmt" "io" "net/http" "net/url" + "os" gopath "path" "runtime/debug" "strings" "time" - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/dagutils" - "github.com/ipfs/go-ipfs/namesys/resolve" - "github.com/dustin/go-humanize" "github.com/ipfs/go-cid" - chunker "github.com/ipfs/go-ipfs-chunker" files "github.com/ipfs/go-ipfs-files" - ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-mfs" "github.com/ipfs/go-path" "github.com/ipfs/go-path/resolver" - ft "github.com/ipfs/go-unixfs" - "github.com/ipfs/go-unixfs/importer" coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p-core/routing" @@ -40,27 +33,36 @@ const ( // gatewayHandler is a HTTP handler that serves IPFS objects (accessible by default at /ipfs/) // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link) type gatewayHandler struct { - node *core.IpfsNode config GatewayConfig api coreiface.CoreAPI } -func newGatewayHandler(n *core.IpfsNode, c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler { +func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler { i := &gatewayHandler{ - node: n, config: c, api: api, } return i } -// TODO(cryptix): find these helpers somewhere else -func (i *gatewayHandler) newDagFromReader(r io.Reader) (ipld.Node, error) { - // TODO(cryptix): change and remove this helper once PR1136 is merged - // return ufs.AddFromReader(i.node, r.Body) - return importer.BuildDagFromReader( - i.node.DAG, - chunker.DefaultSplitter(r)) +func parseIpfsPath(p string) (cid.Cid, string, error) { + rootPath, err := path.ParsePath(p) + if err != nil { + return cid.Cid{}, "", err + } + + // Check the path. + rsegs := rootPath.Segments() + if rsegs[0] != "ipfs" { + return cid.Cid{}, "", fmt.Errorf("WritableGateway: only ipfs paths supported") + } + + rootCid, err := cid.Decode(rsegs[1]) + if err != nil { + return cid.Cid{}, "", err + } + + return rootCid, path.Join(rsegs[2:]), nil } func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -160,10 +162,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // Resolve path to the final DAG node for the ETag resolvedPath, err := i.api.ResolvePath(r.Context(), parsedPath) - if err == coreiface.ErrOffline && !i.node.IsOnline { + switch err { + case nil: + case coreiface.ErrOffline: webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusServiceUnavailable) return - } else if err != nil { + default: webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound) return } @@ -395,102 +399,91 @@ func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) { } func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { - rootPath, err := path.ParsePath(r.URL.Path) + ctx := r.Context() + ds := i.api.Dag() + + // Parse the path + rootCid, newPath, err := parseIpfsPath(r.URL.Path) if err != nil { - webError(w, "putHandler: IPFS path not valid", err, http.StatusBadRequest) + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest) + return + } + if newPath == "" || newPath == "/" { + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest) + return + } + newDirectory, newFileName := gopath.Split(newPath) + + // Resolve the old root. + + rnode, err := ds.Get(ctx, rootCid) + if err != nil { + webError(w, "WritableGateway: Could not create DAG from request", err, http.StatusInternalServerError) return } - rsegs := rootPath.Segments() - if rsegs[0] == ipnsPathPrefix { - webError(w, "putHandler: updating named entries not supported", errors.New("WritableGateway: ipns put not supported"), http.StatusBadRequest) + pbnd, ok := rnode.(*dag.ProtoNode) + if !ok { + webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) return } - var newnode ipld.Node - if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" { - newnode = ft.EmptyDirNode() - } else { - putNode, err := i.newDagFromReader(r.Body) - if err != nil { - webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError) - return - } - newnode = putNode + // Create the new file. + newFilePath, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(r.Body)) + if err != nil { + webError(w, "WritableGateway: could not create DAG from request", err, http.StatusInternalServerError) + return } - var newPath string - if len(rsegs) > 1 { - newPath = path.Join(rsegs[2:]) + newFile, err := ds.Get(ctx, newFilePath.Cid()) + if err != nil { + webError(w, "WritableGateway: failed to resolve new file", err, http.StatusInternalServerError) + return } - var newcid cid.Cid - rnode, err := resolve.Resolve(r.Context(), i.node.Namesys, i.node.Resolver, rootPath) - switch ev := err.(type) { - case resolver.ErrNoLink: - // ev.Node < node where resolve failed - // ev.Name < new link - // but we need to patch from the root - c, err := cid.Decode(rsegs[1]) + // Patch the new file into the old root. + + root, err := mfs.NewRoot(ctx, ds, pbnd, nil) + if err != nil { + webError(w, "WritableGateway: failed to create MFS root", err, http.StatusBadRequest) + return + } + + if newDirectory != "" { + err := mfs.Mkdir(root, newDirectory, mfs.MkdirOpts{Mkparents: true, Flush: false}) if err != nil { - webError(w, "putHandler: bad input path", err, http.StatusBadRequest) - return - } - - rnode, err := i.node.DAG.Get(r.Context(), c) - if err != nil { - webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError) - return - } - - pbnd, ok := rnode.(*dag.ProtoNode) - if !ok { - webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) - return - } - - e := dagutils.NewDagEditor(pbnd, i.node.DAG) - err = e.InsertNodeAtPath(r.Context(), newPath, newnode, ft.EmptyDirNode) - if err != nil { - webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError) - return - } - - nnode, err := e.Finalize(r.Context(), i.node.DAG) - if err != nil { - webError(w, "putHandler: could not get node", err, http.StatusInternalServerError) - return - } - - newcid = nnode.Cid() - - case nil: - pbnd, ok := rnode.(*dag.ProtoNode) - if !ok { - webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) - return - } - - pbnewnode, ok := newnode.(*dag.ProtoNode) - if !ok { - webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) - return - } - - // object set-data case - pbnd.SetData(pbnewnode.Data()) - - newcid = pbnd.Cid() - err = i.node.DAG.Add(r.Context(), pbnd) - if err != nil { - nnk := newnode.Cid() - webError(w, fmt.Sprintf("putHandler: Could not add newnode(%q) to root(%q)", nnk.String(), newcid.String()), err, http.StatusInternalServerError) + webError(w, "WritableGateway: failed to create MFS directory", err, http.StatusInternalServerError) return } + } + dirNode, err := mfs.Lookup(root, newDirectory) + if err != nil { + webError(w, "WritableGateway: failed to lookup directory", err, http.StatusInternalServerError) + return + } + dir, ok := dirNode.(*mfs.Directory) + if !ok { + http.Error(w, "WritableGateway: target directory is not a directory", http.StatusBadRequest) + return + } + err = dir.Unlink(newFileName) + switch err { + case os.ErrNotExist, nil: default: - webError(w, "could not resolve root DAG", ev, http.StatusInternalServerError) + webError(w, "WritableGateway: failed to replace existing file", err, http.StatusBadRequest) return } + err = dir.AddChild(newFileName, newFile) + if err != nil { + webError(w, "WritableGateway: failed to link file into directory", err, http.StatusInternalServerError) + return + } + nnode, err := root.GetDirectory().GetNode() + if err != nil { + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError) + return + } + newcid := nnode.Cid() i.addUserHeaders(w) // ok, _now_ write user's headers. w.Header().Set("IPFS-Hash", newcid.String()) @@ -498,91 +491,75 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { } func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) { - urlPath := r.URL.Path + ctx := r.Context() - p, err := path.ParsePath(urlPath) + // parse the path + + rootCid, newPath, err := parseIpfsPath(r.URL.Path) if err != nil { - webError(w, "failed to parse path", err, http.StatusBadRequest) + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest) return } - - c, components, err := path.SplitAbsPath(p) - if err != nil { - webError(w, "Could not split path", err, http.StatusInternalServerError) + if newPath == "" || newPath == "/" { + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest) return } + directory, filename := gopath.Split(newPath) - pathNodes, err := i.resolvePathComponents(r.Context(), c, components) + // lookup the root + + rootNodeIPLD, err := i.api.Dag().Get(ctx, rootCid) if err != nil { - webError(w, "Could not resolve path components", err, http.StatusBadRequest) + webError(w, "WritableGateway: failed to resolve root CID", err, http.StatusInternalServerError) return } - - pbnd, ok := pathNodes[len(pathNodes)-1].(*dag.ProtoNode) + rootNode, ok := rootNodeIPLD.(*dag.ProtoNode) if !ok { - webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) + http.Error(w, "WritableGateway: empty path", http.StatusInternalServerError) return } - // TODO(cyrptix): assumes len(pathNodes) > 1 - not found is an error above? - err = pbnd.RemoveNodeLink(components[len(components)-1]) + // construct the mfs root + + root, err := mfs.NewRoot(ctx, i.api.Dag(), rootNode, nil) if err != nil { - webError(w, "Could not delete link", err, http.StatusBadRequest) + webError(w, "WritableGateway: failed to construct the MFS root", err, http.StatusBadRequest) return } - var newnode *dag.ProtoNode = pbnd - for j := len(pathNodes) - 2; j >= 0; j-- { - if err := i.node.DAG.Add(r.Context(), newnode); err != nil { - webError(w, "Could not add node", err, http.StatusInternalServerError) - return - } + // lookup the parent directory - pathpb, ok := pathNodes[j].(*dag.ProtoNode) - if !ok { - webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) - return - } - - newnode, err = pathpb.UpdateNodeLink(components[j], newnode) - if err != nil { - webError(w, "Could not update node links", err, http.StatusInternalServerError) - return - } - } - - if err := i.node.DAG.Add(r.Context(), newnode); err != nil { - webError(w, "Could not add root node", err, http.StatusInternalServerError) + parentNode, err := mfs.Lookup(root, directory) + if err != nil { + webError(w, "WritableGateway: failed to look up parent", err, http.StatusInternalServerError) return } - // Redirect to new path - ncid := newnode.Cid() + parent, ok := parentNode.(*mfs.Directory) + if !ok { + http.Error(w, "WritableGateway: parent is not a directory", http.StatusInternalServerError) + return + } + + // delete the file + + switch parent.Unlink(filename) { + case nil, os.ErrNotExist: + default: + webError(w, "WritableGateway: failed to remove file", err, http.StatusInternalServerError) + return + } + + nnode, err := root.GetDirectory().GetNode() + if err != nil { + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError) + } + ncid := nnode.Cid() i.addUserHeaders(w) // ok, _now_ write user's headers. w.Header().Set("IPFS-Hash", ncid.String()) - http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), path.Join(components[:len(components)-1])), http.StatusCreated) -} - -func (i *gatewayHandler) resolvePathComponents( - ctx context.Context, - c cid.Cid, - components []string, -) ([]ipld.Node, error) { - tctx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - - rootnd, err := i.node.Resolver.DAG.Get(tctx, c) - if err != nil { - return nil, fmt.Errorf("Could not resolve root object: %s", err) - } - - pathNodes, err := i.node.Resolver.ResolveLinks(tctx, rootnd, components[:len(components)-1]) - if err != nil { - return nil, fmt.Errorf("Could not resolve parent object: %s", err) - } - - return pathNodes, nil + // note: StatusCreated is technically correct here as we created a new resource. + http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), directory), http.StatusCreated) } func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) { diff --git a/go.mod b/go.mod index 3dc99cd7b..9c0b3361a 100644 --- a/go.mod +++ b/go.mod @@ -45,12 +45,12 @@ require ( github.com/ipfs/go-ipld-git v0.0.2 github.com/ipfs/go-ipns v0.0.1 github.com/ipfs/go-log v0.0.1 - github.com/ipfs/go-merkledag v0.2.0 + github.com/ipfs/go-merkledag v0.2.3 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.0 github.com/ipfs/go-path v0.0.7 - github.com/ipfs/go-unixfs v0.2.0 + github.com/ipfs/go-unixfs v0.2.1 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/hang-fds v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.1.0 diff --git a/go.sum b/go.sum index f21188d11..06d4b5b02 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJ github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.0 h1:EAjIQCgZ6/DnOAlKY3+59j72FD9BsYtNaCRSmN0xIbU= github.com/ipfs/go-merkledag v0.2.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= +github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= @@ -320,8 +322,8 @@ github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.0.8 h1:AHahQ+gdNZd9BhKVLf8XP1EWeKa78eTzYgCygp7N/Pg= github.com/ipfs/go-unixfs v0.0.8/go.mod h1:cK2vDJ7L4YnWB6oLefpVNesgx0x/zPTRVDw6B4Y+03U= -github.com/ipfs/go-unixfs v0.2.0 h1:mfdI8rgsEifWfhLECrH2WphHvslNoPbdvlmsJ05Fu0M= -github.com/ipfs/go-unixfs v0.2.0/go.mod h1:sy/j20FKUxdUYOl6GZb3wsQ593C2xk5r1i8U7W+5iio= +github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0= +github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/hang-fds v0.0.1 h1:KGAxiGtJPT3THVRNT6yxgpdFPeX4ZemUjENOt6NlOn4= diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index 71775d5d1..abd2d57ff 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -54,17 +54,8 @@ test_expect_success "We can HTTP GET file just created" ' test_cmp infile outfile ' -test_expect_success "HTTP PUT empty directory" ' - URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" && - echo "PUT $URL" && - curl -svX PUT "$URL" 2>curl_putEmpty.out && - cat curl_putEmpty.out && - grep "Ipfs-Hash: $HASH_EMPTY_DIR" curl_putEmpty.out && - grep "Location: /ipfs/$HASH_EMPTY_DIR" curl_putEmpty.out && - grep "HTTP/1.1 201 Created" curl_putEmpty.out -' - test_expect_success "HTTP GET empty directory" ' + URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" && echo "GET $URL" && curl -so outfile "$URL" 2>curl_getEmpty.out && grep "Index of /ipfs/$HASH_EMPTY_DIR/" outfile @@ -105,6 +96,24 @@ test_expect_success "We can HTTP GET file just updated" ' test_cmp infile2 outfile2 ' +test_expect_success "HTTP PUT to replace a directory" ' + echo "$RANDOM" >infile3 && + URL="http://localhost:$port/ipfs/$HASH/test" && + echo "PUT $URL" && + curl -svX PUT --data-binary @infile3 "$URL" 2>curl_putOverDirectory.out && + grep "HTTP/1.1 201 Created" curl_putOverDirectory.out && + LOCATION=$(grep Location curl_putOverDirectory.out) && + HASH=$(expr "$LOCATION" : "< Location: /ipfs/\(.*\)/test") +' + +test_expect_success "We can HTTP GET file just put over a directory" ' + URL="http://localhost:$port/ipfs/$HASH/test" && + echo "GET $URL" && + curl -svo outfile3 "$URL" 2>curl_getOverDirectory.out && + test_cmp infile3 outfile3 +' + + test_kill_ipfs_daemon test_done From 2c222741d1454df609d147ce417f16029f97d95d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Jul 2019 19:26:06 -0700 Subject: [PATCH 2/4] fix: update for merkledag API changes --- core/commands/dht.go | 2 +- core/commands/pin.go | 8 ++------ core/coreapi/dht.go | 2 +- core/coreapi/pin.go | 10 ++++------ dagutils/diffenum.go | 2 +- pin/gc/gc.go | 2 +- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/core/commands/dht.go b/core/commands/dht.go index 2bc0c790d..8ee5d7d9d 100644 --- a/core/commands/dht.go +++ b/core/commands/dht.go @@ -326,7 +326,7 @@ func provideKeysRec(ctx context.Context, r routing.Routing, dserv ipld.DAGServic for _, c := range cids { kset := cid.NewSet() - err := dag.WalkParallel(ctx, dag.GetLinksDirect(dserv), c, kset.Visit) + err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, kset.Visit) if err != nil { return err } diff --git a/core/commands/pin.go b/core/commands/pin.go index 9392e7b90..71cb4f349 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -503,11 +503,7 @@ func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(val if typeStr == "indirect" || typeStr == "all" { for _, k := range n.Pinning.RecursiveKeys() { var visitErr error - err := dag.WalkParallelDepth(req.Context, dag.GetLinksWithDAG(n.DAG), k, 0, func(c cid.Cid, depth int) bool { - if depth == 0 { - // skip it without visiting it. - return true - } + err := dag.Walk(req.Context, dag.GetLinksWithDAG(n.DAG), k, func(c cid.Cid) bool { r := keys.Visit(c) if r { err := emit(&PinLsOutputWrapper{ @@ -521,7 +517,7 @@ func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(val } } return r - }) + }, dag.SkipRoot(), dag.Concurrent()) if visitErr != nil { return visitErr diff --git a/core/coreapi/dht.go b/core/coreapi/dht.go index 902a26cc0..892e75576 100644 --- a/core/coreapi/dht.go +++ b/core/coreapi/dht.go @@ -114,7 +114,7 @@ func provideKeysRec(ctx context.Context, r routing.Routing, bs blockstore.Blocks go func() { dserv := dag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) for _, c := range cids { - err := dag.WalkParallel(ctx, dag.GetLinksDirect(dserv), c, provided.Visitor(ctx)) + err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, provided.Visitor(ctx)) if err != nil { errCh <- err } diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 7955ff081..15951687b 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -209,12 +209,10 @@ func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pi if typeStr == "indirect" || typeStr == "all" { set := cid.NewSet() for _, k := range api.pinning.RecursiveKeys() { - err := merkledag.WalkParallelDepth( - ctx, merkledag.GetLinksWithDAG(api.dag), k, 0, - func(c cid.Cid, depth int) bool { - // don't visit the root node, that doesn't count. - return depth == 0 || set.Visit(c) - }, + err := merkledag.Walk( + ctx, merkledag.GetLinksWithDAG(api.dag), k, + set.Visit, + merkledag.SkipRoot(), merkledag.Concurrent(), ) if err != nil { return nil, err diff --git a/dagutils/diffenum.go b/dagutils/diffenum.go index 3afcce4f5..fdab772c8 100644 --- a/dagutils/diffenum.go +++ b/dagutils/diffenum.go @@ -40,7 +40,7 @@ func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid) if sset.Has(c.aft) { continue } - err := mdag.WalkParallel(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit) + err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent()) if err != nil { return err } diff --git a/pin/gc/gc.go b/pin/gc/gc.go index ad3c4149d..e03072770 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -171,7 +171,7 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots for _, c := range roots { // Walk recursively walks the dag and adds the keys to the given set - err := dag.WalkParallel(ctx, verifyGetLinks, c, set.Visit) + err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent()) if err != nil { err = verboseCidError(err) From e282aa1480bec1033e4d101203f51db21c43179b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Jul 2019 19:55:46 -0700 Subject: [PATCH 3/4] sharness: make sure the writable gateway produces the correct hash --- test/sharness/t0111-gateway-writeable.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index abd2d57ff..b6d8d8474 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -54,6 +54,11 @@ test_expect_success "We can HTTP GET file just created" ' test_cmp infile outfile ' +test_expect_success "We got the correct hash" ' + ADD_HASH="/ipfs/$(ipfs add -q infile)" && + test "x$ADD_HASH" = "x$HASH" || test_fsh echo "$ADD_HASH != $HASH" +' + test_expect_success "HTTP GET empty directory" ' URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" && echo "GET $URL" && From 50a1113b929f0d813309c516bd9cc3f44ac5bbb9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Jul 2019 22:47:57 -0700 Subject: [PATCH 4/4] sharness: make sure putting to IPNS fails --- test/sharness/t0111-gateway-writeable.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index b6d8d8474..423151c6a 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -118,6 +118,14 @@ test_expect_success "We can HTTP GET file just put over a directory" ' test_cmp infile3 outfile3 ' +test_expect_success "HTTP PUT to /ipns fails" ' + PEERID=`ipfs id --format=""` && + URL="http://localhost:$port/ipns/$PEERID/test.txt" && + echo "PUT $URL" && + curl -svX PUT --data-binary @infile1 "$URL" 2>curl_putIpns.out && + grep "HTTP/1.1 400 Bad Request" curl_putIpns.out +' + test_kill_ipfs_daemon