From 202081523fdc8bc54b16e5e4447b221586febef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:12:10 +0100 Subject: [PATCH 001/112] Setup repo This commit was moved from ipfs/go-ipfs-http-client@4e7edce41d91825f0ea21dec426de2e128db5b38 --- client/httpapi/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 client/httpapi/README.md diff --git a/client/httpapi/README.md b/client/httpapi/README.md new file mode 100644 index 000000000..ab37a4cf8 --- /dev/null +++ b/client/httpapi/README.md @@ -0,0 +1,27 @@ +# go-ipfs-http-api + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) + +> IPFS CoreAPI implementation using HTTP API + +## Documentation + +https://godoc.org/github.com/ipfs/go-ipfs-http-api + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +### Want to hack on IPFS? + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT From 6d85aff407d3fb3eb6d2549fbac94615fd147b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:13:15 +0100 Subject: [PATCH 002/112] Initial structure, path stuff This commit was moved from ipfs/go-ipfs-http-client@93943f7f5671948b0d2aee19c30cf689307faf13 --- client/httpapi/api.go | 127 +++++++++++++++++++++++++++++ client/httpapi/name.go | 35 ++++++++ client/httpapi/path.go | 48 +++++++++++ client/httpapi/request.go | 34 ++++++++ client/httpapi/requestbuilder.go | 100 +++++++++++++++++++++++ client/httpapi/response.go | 132 +++++++++++++++++++++++++++++++ 6 files changed, 476 insertions(+) create mode 100644 client/httpapi/api.go create mode 100644 client/httpapi/name.go create mode 100644 client/httpapi/path.go create mode 100644 client/httpapi/request.go create mode 100644 client/httpapi/requestbuilder.go create mode 100644 client/httpapi/response.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go new file mode 100644 index 000000000..82e030350 --- /dev/null +++ b/client/httpapi/api.go @@ -0,0 +1,127 @@ +package httpapi + +import ( + "github.com/pkg/errors" + "io/ioutil" + gohttp "net/http" + "os" + "path" + "strings" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + homedir "github.com/mitchellh/go-homedir" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" + ) + +const ( + DefaultPathName = ".ipfs" + DefaultPathRoot = "~/" + DefaultPathName + DefaultApiFile = "api" + EnvDir = "IPFS_PATH" +) + +var ErrNotImplemented = errors.New("not implemented") + +type HttpApi struct { + url string + httpcli *gohttp.Client +} + +func NewLocalApi() iface.CoreAPI { + baseDir := os.Getenv(EnvDir) + if baseDir == "" { + baseDir = DefaultPathRoot + } + + baseDir, err := homedir.Expand(baseDir) + if err != nil { + return nil + } + + apiFile := path.Join(baseDir, DefaultApiFile) + + if _, err := os.Stat(apiFile); err != nil { + return nil + } + + api, err := ioutil.ReadFile(apiFile) + if err != nil { + return nil + } + + return NewApi(strings.TrimSpace(string(api))) +} + +func NewApi(url string) *HttpApi { + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + } + + return NewApiWithClient(url, c) +} + +func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { + if a, err := ma.NewMultiaddr(url); err == nil { + _, host, err := manet.DialArgs(a) + if err == nil { + url = host + } + } + + return &HttpApi{ + url: url, + httpcli: c, + } +} + +func (api *HttpApi) request(command string, args ...string) *RequestBuilder { + return &RequestBuilder{ + command: command, + args: args, + shell: api, + } +} + +func (api *HttpApi) Unixfs() iface.UnixfsAPI { + return nil +} + +func (api *HttpApi) Block() iface.BlockAPI { + return nil +} + +func (api *HttpApi) Dag() iface.DagAPI { + return nil +} + +func (api *HttpApi) Name() iface.NameAPI { + return (*NameAPI)(api) +} + +func (api *HttpApi) Key() iface.KeyAPI { + return nil +} + +func (api *HttpApi) Pin() iface.PinAPI { + return nil +} + +func (api *HttpApi) Object() iface.ObjectAPI { + return nil +} + +func (api *HttpApi) Dht() iface.DhtAPI { + return nil +} + +func (api *HttpApi) Swarm() iface.SwarmAPI { + return nil +} + +func (api *HttpApi) PubSub() iface.PubSubAPI { + return nil +} diff --git a/client/httpapi/name.go b/client/httpapi/name.go new file mode 100644 index 000000000..41426ef57 --- /dev/null +++ b/client/httpapi/name.go @@ -0,0 +1,35 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type NameAPI HttpApi + +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { + // TODO: options! + + req := api.core().request("name/resolve") + req.Arguments(name) + + var out struct{ Path string } + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + + return iface.ParsePath(out.Path) +} + +func (api *NameAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/path.go b/client/httpapi/path.go new file mode 100644 index 000000000..28656fbd4 --- /dev/null +++ b/client/httpapi/path.go @@ -0,0 +1,48 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { + var out struct { + Cid cid.Cid + RemPath string + } + + //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help + + var err error + if path.Namespace() == "ipns" { + if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + return nil, err + } + } + + if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + // TODO: + ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + if err != nil { + return nil, err + } + + return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil +} + +func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { + return nil, ErrNotImplemented +} diff --git a/client/httpapi/request.go b/client/httpapi/request.go new file mode 100644 index 000000000..58c61ac67 --- /dev/null +++ b/client/httpapi/request.go @@ -0,0 +1,34 @@ +package httpapi + +import ( + "context" + "io" + "strings" +) + +type Request struct { + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string +} + +func NewRequest(ctx context.Context, url, command string, args ...string) *Request { + if !strings.HasPrefix(url, "http") { + url = "http://" + url + } + + opts := map[string]string{ + "encoding": "json", + "stream-channels": "true", + } + return &Request{ + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), + } +} diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go new file mode 100644 index 000000000..9ccc8cf97 --- /dev/null +++ b/client/httpapi/requestbuilder.go @@ -0,0 +1,100 @@ +package httpapi + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" +) + +// RequestBuilder is an IPFS commands request builder. +type RequestBuilder struct { + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader + + shell *HttpApi +} + +// Arguments adds the arguments to the args. +func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { + r.args = append(r.args, args...) + return r +} + +// BodyString sets the request body to the given string. +func (r *RequestBuilder) BodyString(body string) *RequestBuilder { + return r.Body(strings.NewReader(body)) +} + +// BodyBytes sets the request body to the given buffer. +func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { + return r.Body(bytes.NewReader(body)) +} + +// Body sets the request body to the given reader. +func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { + r.body = body + return r +} + +// Option sets the given option. +func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { + var s string + switch v := value.(type) { + case bool: + s = strconv.FormatBool(v) + case string: + s = v + case []byte: + s = string(v) + default: + // slow case. + s = fmt.Sprint(value) + } + if r.opts == nil { + r.opts = make(map[string]string, 1) + } + r.opts[key] = s + return r +} + +// Header sets the given header. +func (r *RequestBuilder) Header(name, value string) *RequestBuilder { + if r.headers == nil { + r.headers = make(map[string]string, 1) + } + r.headers[name] = value + return r +} + +// Send sends the request and return the response. +func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + req := NewRequest(ctx, r.shell.url, r.command, r.args...) + req.Opts = r.opts + req.Headers = r.headers + req.Body = r.body + return req.Send(r.shell.httpcli) +} + +// Exec sends the request a request and decodes the response. +func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { + httpRes, err := r.Send(ctx) + if err != nil { + return err + } + + if res == nil { + httpRes.Close() + if httpRes.Error != nil { + return httpRes.Error + } + return nil + } + + return httpRes.Decode(res) +} diff --git a/client/httpapi/response.go b/client/httpapi/response.go new file mode 100644 index 000000000..27709769b --- /dev/null +++ b/client/httpapi/response.go @@ -0,0 +1,132 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + files "github.com/ipfs/go-ipfs-files" +) + +type Response struct { + Output io.ReadCloser + Error *Error +} + +func (r *Response) Close() error { + if r.Output != nil { + // always drain output (response body) + ioutil.ReadAll(r.Output) + return r.Output.Close() + } + return nil +} + +func (r *Response) Decode(dec interface{}) error { + defer r.Close() + if r.Error != nil { + return r.Error + } + + return json.NewDecoder(r.Output).Decode(dec) +} + +type Error struct { + Command string + Message string + Code int +} + +func (e *Error) Error() string { + var out string + if e.Command != "" { + out = e.Command + ": " + } + if e.Code != 0 { + out = fmt.Sprintf("%s%d: ", out, e.Code) + } + return out + e.Message +} + +func (r *Request) Send(c *http.Client) (*Response, error) { + url := r.getURL() + req, err := http.NewRequest("POST", url, r.Body) + if err != nil { + return nil, err + } + + // Add any headers that were supplied via the RequestBuilder. + for k, v := range r.Headers { + req.Header.Add(k, v) + } + + if fr, ok := r.Body.(*files.MultiFileReader); ok { + req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) + req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + } + + resp, err := c.Do(req) + if err != nil { + return nil, err + } + + contentType := resp.Header.Get("Content-Type") + parts := strings.Split(contentType, ";") + contentType = parts[0] + + nresp := new(Response) + + nresp.Output = resp.Body + if resp.StatusCode >= http.StatusBadRequest { + e := &Error{ + Command: r.Command, + } + switch { + case resp.StatusCode == http.StatusNotFound: + e.Message = "command not found" + case contentType == "text/plain": + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = string(out) + case contentType == "application/json": + if err = json.NewDecoder(resp.Body).Decode(e); err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) + } + default: + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out) + } + nresp.Error = e + nresp.Output = nil + + // drain body and close + ioutil.ReadAll(resp.Body) + resp.Body.Close() + } + + return nresp, nil +} + +func (r *Request) getURL() string { + + values := make(url.Values) + for _, arg := range r.Args { + values.Add("arg", arg) + } + for k, v := range r.Opts { + values.Add(k, v) + } + + return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode()) +} From 35c271e3d88725a9ab25be40c1837435de110a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:40:02 +0100 Subject: [PATCH 003/112] wip notice in readme This commit was moved from ipfs/go-ipfs-http-client@e823507553ca82115e303e1ba26f76b83f5bd12c --- client/httpapi/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index ab37a4cf8..dd09142e3 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,6 +8,8 @@ > IPFS CoreAPI implementation using HTTP API +This project is WIP, use https://github.com/ipfs/go-ipfs-api for now + ## Documentation https://godoc.org/github.com/ipfs/go-ipfs-http-api From 6c927fd9624807e124f0376445c41daaa975fbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:38:11 +0100 Subject: [PATCH 004/112] Partial ipld node impl This commit was moved from ipfs/go-ipfs-http-client@df916c7849d6cfd7f479c4fe80c0f75d6705a0fc --- client/httpapi/api.go | 6 +-- client/httpapi/block.go | 40 ++++++++++++++ client/httpapi/ipldnode.go | 107 +++++++++++++++++++++++++++++++++++++ client/httpapi/path.go | 2 +- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/block.go create mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 82e030350..cd3fb9fd0 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,7 @@ package httpapi import ( - "github.com/pkg/errors" + "errors" "io/ioutil" gohttp "net/http" "os" @@ -12,7 +12,7 @@ import ( homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - ) +) const ( DefaultPathName = ".ipfs" @@ -91,7 +91,7 @@ func (api *HttpApi) Unixfs() iface.UnixfsAPI { } func (api *HttpApi) Block() iface.BlockAPI { - return nil + return (*BlockAPI)(api) } func (api *HttpApi) Dag() iface.DagAPI { diff --git a/client/httpapi/block.go b/client/httpapi/block.go new file mode 100644 index 000000000..8bdb4c502 --- /dev/null +++ b/client/httpapi/block.go @@ -0,0 +1,40 @@ +package httpapi + +import ( + "context" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type BlockAPI HttpApi + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + if err != nil { + return nil, err + } + + //TODO: is close on the reader enough? + //defer resp.Close() + + //TODO: make blockApi return ReadCloser + return resp.Output, resp.Error +} + +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { + return ErrNotImplemented +} + +func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go new file mode 100644 index 000000000..a3dd6204b --- /dev/null +++ b/client/httpapi/ipldnode.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + "io/ioutil" + "strconv" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type ipldNode struct { + ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? + path iface.ResolvedPath + api *HttpApi +} + +func (n *ipldNode) RawData() []byte { + r, err := n.api.Block().Get(n.ctx, n.path) + if err != nil { + panic(err) // TODO: eww, should we add errors too / better ideas? + } + + b, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + return b +} + +func (n *ipldNode) Cid() cid.Cid { + return n.path.Cid() +} + +func (n *ipldNode) String() string { + return fmt.Sprintf("[Block %s]", n.Cid()) +} + +func (n *ipldNode) Loggable() map[string]interface{} { + return nil //TODO: we can't really do better here, can we? +} + +// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. +// We can also give people a choice +func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { + p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) + + var out interface{} + n.api.request("dag/get", p).Exec(n.ctx, &out) + + // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack + for len(path) > 0 { + switch o := out.(type) { + case map[string]interface{}: + v, ok := o[path[0]] + if !ok { + // TODO: ipld links + return nil, nil, errors.New("no element under this path") + } + out = v + case []interface{}: + n, err := strconv.ParseUint(path[0], 10, 32) + if err != nil { + return nil, nil, err + } + if len(o) < int(n) { + return nil, nil, errors.New("no element under this path") + } + out = o[n] + } + path = path[1:] + } + + return out, path, nil +} + +func (n *ipldNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (n *ipldNode) Copy() ipld.Node { + panic("implement me") +} + +func (n *ipldNode) Links() []*ipld.Link { + panic("implement me") +} + +func (n *ipldNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (n *ipldNode) Size() (uint64, error) { + panic("implement me") +} + +var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 28656fbd4..6b6e4b027 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -30,7 +30,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } // TODO: - ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } From 1cd2ec05b7524e921f58750a4b9c8f48ad1a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Dec 2018 16:04:19 +0100 Subject: [PATCH 005/112] Init gx This commit was moved from ipfs/go-ipfs-http-client@e06cddbedd360b1982e414571dd08ab25df38546 --- client/httpapi/ipldnode.go | 8 ++++---- client/httpapi/name.go | 1 + client/httpapi/path.go | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index a3dd6204b..b8e6fba01 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -2,16 +2,16 @@ package httpapi import ( "context" + "errors" "fmt" - "github.com/pkg/errors" - "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" "io/ioutil" "strconv" "github.com/ipfs/go-ipfs/core/coreapi/interface" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) type ipldNode struct { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 41426ef57..7315ac2c3 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 6b6e4b027..5701326fc 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -5,9 +5,9 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { From 16e97bf5e8f80b681c5a8d4b96f406b4b83a8e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 21 Dec 2018 18:44:34 +0100 Subject: [PATCH 006/112] It builds This commit was moved from ipfs/go-ipfs-http-client@a23d794e5fceadb9115d8d0c8466c10e9c5d0891 --- client/httpapi/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cd3fb9fd0..dc9acb9f6 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -78,6 +79,10 @@ func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { } } +func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { + return nil, ErrNotImplemented +} + func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, From fe4c9fd8033c94c5ea707b09e76479a6d018635d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 30 Dec 2018 04:24:09 +0100 Subject: [PATCH 007/112] Skeleton for tests This commit was moved from ipfs/go-ipfs-http-client@dfbe0026ad9438d4e795bbb43debeffb6bb9ca1a --- client/httpapi/api.go | 24 ++++++++--- client/httpapi/api_test.go | 84 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 client/httpapi/api_test.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dc9acb9f6..63ea3aad7 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -35,7 +35,11 @@ func NewLocalApi() iface.CoreAPI { baseDir = DefaultPathRoot } - baseDir, err := homedir.Expand(baseDir) + return NewPathApi(baseDir) +} + +func NewPathApi(p string) iface.CoreAPI { + baseDir, err := homedir.Expand(p) if err != nil { return nil } @@ -51,10 +55,15 @@ func NewLocalApi() iface.CoreAPI { return nil } - return NewApi(strings.TrimSpace(string(api))) + maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) + if err != nil { + return nil + } + + return NewApi(maddr) } -func NewApi(url string) *HttpApi { +func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -62,10 +71,15 @@ func NewApi(url string) *HttpApi { }, } - return NewApiWithClient(url, c) + return NewApiWithClient(a, c) } -func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { + _, url, err := manet.DialArgs(a) + if err != nil { + return nil // TODO: return that error + } + if a, err := ma.NewMultiaddr(url); err == nil { _, host, err := manet.DialArgs(a) if err == nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go new file mode 100644 index 000000000..b925b5c34 --- /dev/null +++ b/client/httpapi/api_test.go @@ -0,0 +1,84 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/iptb/testbed/interfaces" + "io/ioutil" + "os" + "path" + "strconv" + "testing" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" + + local "github.com/ipfs/iptb-plugins/local" + "github.com/ipfs/iptb/cli" + "github.com/ipfs/iptb/testbed" +) + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ + From: "", + NewNode: local.NewNode, + GetAttrList: local.GetAttrList, + GetAttrDesc: local.GetAttrDesc, + PluginName: local.PluginName, + BuiltIn: true, + }, false) + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "httpapi-tb-") + if err != nil { + return nil, err + } + + c := cli.NewCli() + if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + + defer os.Remove(dir) + + defer func() { + _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + }() + }() + + apis := make([]iface.CoreAPI, n) + + for i := range apis { + tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) + + node, err := tb.Node(i) + if err != nil { + return nil, err + } + + attrNode, ok := node.(testbedi.Attribute) + if !ok { + return nil, fmt.Errorf("node does not implement attributes") + } + + pth, err := attrNode.Attr("path") + if err != nil { + return nil, err + } + + apis[i] = NewPathApi(pth) + } + + return apis, nil +} + +func TestHttpApi(t *testing.T) { + tests.TestApi(&NodeProvider{})(t) +} From e8da6e2cf9a8fd40c5fd960a60ce1b1b27ada713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 7 Jan 2019 23:44:43 +0100 Subject: [PATCH 008/112] Block API This commit was moved from ipfs/go-ipfs-http-client@d0c98b870e55bed78fa777793e520f08157deb4f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 11 +++- client/httpapi/block.go | 109 ++++++++++++++++++++++++++++--- client/httpapi/requestbuilder.go | 12 ++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 63ea3aad7..1768e8477 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return (*NameAPI)(api) + return nil } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b925b5c34..e32c456e5 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/iptb/testbed/interfaces" "io/ioutil" "os" "path" @@ -16,6 +15,7 @@ import ( local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" + "github.com/ipfs/iptb/testbed/interfaces" ) type NodeProvider struct{} @@ -39,7 +39,14 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := cli.NewCli() - if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + + initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} + if err := c.Run(initArgs); err != nil { + return nil, err + } + + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 8bdb4c502..185aa0a42 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -1,17 +1,69 @@ package httpapi import ( + "bytes" "context" + "errors" + "fmt" "io" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + mh "github.com/multiformats/go-multihash" ) type BlockAPI HttpApi -func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { - return nil, ErrNotImplemented +type blockStat struct { + Key string + BSize int `json:"Size"` +} + +func (s *blockStat) Size() int { + return s.BSize +} + +func (s *blockStat) valid() (iface.ResolvedPath, error) { + c, err := cid.Parse(s.Key) + if err != nil { + return nil, err + } + + return iface.IpldPath(c), nil +} + +func (s *blockStat) Path() iface.ResolvedPath { + p, _ := s.valid() + return p +} + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { + options, _, err := caopts.BlockPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("block/put"). + Option("mhtype", mht). + Option("mhlen", options.MhLength). + Option("format", options.Codec). + FileBody(r) + + var out blockStat + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { @@ -19,20 +71,57 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } - //TODO: is close on the reader enough? - //defer resp.Close() + //TODO: make get return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } - //TODO: make blockApi return ReadCloser - return resp.Output, resp.Error + return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { - return ErrNotImplemented +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { + options, err := caopts.BlockRmOptions(opts...) + if err != nil { + return err + } + + removedBlock := struct { + Hash string `json:",omitempty"` + Error string `json:",omitempty"` + }{} + + req := api.core().request("block/rm"). + Option("force", options.Force). + Arguments(p.String()) + + if err := req.Exec(ctx, &removedBlock); err != nil { + return err + } + + if removedBlock.Error != "" { + return errors.New(removedBlock.Error) + } + + return nil } func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { - return nil, ErrNotImplemented + var out blockStat + err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) core() *HttpApi { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 9ccc8cf97..6e5a89ebd 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,8 +5,11 @@ import ( "context" "fmt" "io" + "io/ioutil" "strconv" "strings" + + "github.com/ipfs/go-ipfs-files" ) // RequestBuilder is an IPFS commands request builder. @@ -42,6 +45,15 @@ func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { return r } +// FileBody sets the request body to the given reader wrapped into multipartreader. +func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { + pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + d := files.NewMapDirectory(map[string]files.Node{"": pr}) + r.body = files.NewMultiFileReader(d, false) + + return r +} + // Option sets the given option. func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { var s string From ab89e0abf99a0bc42ee2d8fb55a9b3de79573635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 00:30:22 +0100 Subject: [PATCH 009/112] Partial Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@44696b84f59f6f707858787c9a0750f224596635 --- client/httpapi/api.go | 4 +- client/httpapi/unixfs.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/unixfs.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 1768e8477..099b45123 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -106,7 +106,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { } func (api *HttpApi) Unixfs() iface.UnixfsAPI { - return nil + return (*UnixfsAPI)(api) } func (api *HttpApi) Block() iface.BlockAPI { @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return nil + return (*NameAPI)(api) } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go new file mode 100644 index 000000000..567f60284 --- /dev/null +++ b/client/httpapi/unixfs.go @@ -0,0 +1,95 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type addEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +type UnixfsAPI HttpApi + +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { + options, _, err := caopts.UnixfsAddOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("add"). + Option("hash", mht). + Option("chunker", options.Chunker). + Option("cid-version", options.CidVersion). + //Option("", options.Events). + Option("fscache", options.FsCache). + Option("hidden", options.Hidden). + Option("inline", options.Inline). + Option("inline-limit", options.InlineLimit). + Option("nocopy", options.NoCopy). + Option("only-hash", options.OnlyHash). + Option("pin", options.Pin). + //Option("", options.Progress). + Option("silent", options.Silent). + Option("stdin-name", options.StdinName). + Option("wrap-with-directory", options.Wrap). + Option("quieter", true) // TODO: rm after event impl + + if options.RawLeavesSet { + req.Option("raw-leaves", options.RawLeaves) + } + + switch options.Layout { + case caopts.BalancedLayout: + // noop, default + case caopts.TrickleLayout: + req.Option("trickle", true) + } + + switch c := f.(type) { + case files.Directory: + req.Body(files.NewMultiFileReader(c, false)) + case files.File: + req.Body(c) + } + + var out addEvent + if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { + panic("implement me") +} + +func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *UnixfsAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cf74d391603457ad626b7f6b27ff0655bc946ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:07:21 +0100 Subject: [PATCH 010/112] Partial Key API, ApiAddr funcion This commit was moved from ipfs/go-ipfs-http-client@0ffdef159261e19cfd76edabb394179aa9295003 --- client/httpapi/api.go | 13 ++++++- client/httpapi/key.go | 86 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/key.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 099b45123..d104d98df 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,6 +29,7 @@ type HttpApi struct { httpcli *gohttp.Client } +//TODO: Return errors here func NewLocalApi() iface.CoreAPI { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -39,6 +40,14 @@ func NewLocalApi() iface.CoreAPI { } func NewPathApi(p string) iface.CoreAPI { + a := ApiAddr(p) + if a == nil { + return nil + } + return NewApi(a) +} + +func ApiAddr(p string) ma.Multiaddr { baseDir, err := homedir.Expand(p) if err != nil { return nil @@ -60,7 +69,7 @@ func NewPathApi(p string) iface.CoreAPI { return nil } - return NewApi(maddr) + return maddr } func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? @@ -122,7 +131,7 @@ func (api *HttpApi) Name() iface.NameAPI { } func (api *HttpApi) Key() iface.KeyAPI { - return nil + return (*KeyAPI)(api) } func (api *HttpApi) Pin() iface.PinAPI { diff --git a/client/httpapi/key.go b/client/httpapi/key.go new file mode 100644 index 000000000..87b573f98 --- /dev/null +++ b/client/httpapi/key.go @@ -0,0 +1,86 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type KeyAPI HttpApi + +type keyOutput struct { + JName string `json:"Name"` + Id string +} + +func (k *keyOutput) Name() string { + return k.JName +} + +func (k *keyOutput) Path() iface.Path { + p, _ := iface.ParsePath("/ipns/" + k.Id) + return p +} + +func (k *keyOutput) ID() peer.ID { + p, _ := peer.IDB58Decode(k.Id) + return p +} + +func (k *keyOutput) valid() error { + _, err := peer.IDB58Decode(k.Id) + return err +} + + +func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { + options, err := caopts.KeyGenerateOptions(opts...) + if err != nil { + return nil, err + } + + var out keyOutput + err = api.core().request("key/gen", name). + Option("type", options.Algorithm). + Option("size", options.Size). + Exec(ctx, &out) + if err != nil { + return nil, err + } + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { + panic("implement me") +} + +func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { + var id struct{ID string} + if err := api.core().request("id").Exec(ctx, &id); err != nil { + return nil, err + } + + out := keyOutput{JName: "self", Id: id.ID} + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c236393733bf3491c45002d67189ba78303332a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:08:18 +0100 Subject: [PATCH 011/112] Connect test swarms, don't compress api calls This commit was moved from ipfs/go-ipfs-http-client@c6472d9b8286c932492db4d6bf03f890bf97759c --- client/httpapi/api_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index e32c456e5..02c7830bb 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + gohttp "net/http" "os" "path" "strconv" @@ -50,6 +51,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + if n > 1 { + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + if err := c.Run(connectArgs); err != nil { + return nil, err + } + } + go func() { <-ctx.Done() @@ -80,7 +88,18 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - apis[i] = NewPathApi(pth) + a := ApiAddr(pth) + if a == nil { + return nil, fmt.Errorf("nil addr for node") + } + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(a, c) } return apis, nil From 7861315f49398e20e7e1189671e42d0a16adbef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:00 +0100 Subject: [PATCH 012/112] Wrap single files in Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@16f77b24a1b150e7ec624d5675ded478ac6cb853 --- client/httpapi/unixfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 567f60284..21f75f85a 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -66,7 +66,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix case files.Directory: req.Body(files.NewMultiFileReader(c, false)) case files.File: - req.Body(c) + d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) } var out addEvent From f638bae3a960a74d284503c30086f0bbffbe73c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:17 +0100 Subject: [PATCH 013/112] implement .Name This commit was moved from ipfs/go-ipfs-http-client@e19e5f54e48d0909796add6d57e3e74e84eba85e --- client/httpapi/name.go | 72 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 7315ac2c3..fbc440b96 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,26 +2,80 @@ package httpapi import ( "context" + "fmt" + "github.com/ipfs/go-ipfs/namesys/opts" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type NameAPI HttpApi -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { +type ipnsEntry struct { + JName string `json:"Name"` + JValue string `json:"Value"` +} + +func (e *ipnsEntry) valid() (iface.Path, error) { + return iface.ParsePath(e.JValue) +} + +func (e *ipnsEntry) Name() string { + return e.JName +} + +func (e *ipnsEntry) Value() iface.Path { + p, _ := e.valid() + return p +} + +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { + options, err := caopts.NamePublishOptions(opts...) + if err != nil { + return nil, err + } + + req := api.core().request("name/publish", p.String()). + Option("key", options.Key). + Option("allow-offline", options.AllowOffline). + Option("lifetime", options.ValidTime.String()). + Option("resolve", false) + + if options.TTL != nil { + req.Option("ttl", options.TTL.String()) + } + + var out ipnsEntry + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { return nil, ErrNotImplemented } -func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { - return nil, ErrNotImplemented -} +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { - // TODO: options! + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } - req := api.core().request("name/resolve") - req.Arguments(name) + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout.String()) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { From 1acf4163902ae3b9e0e182761deafee5e67415ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:53:24 +0100 Subject: [PATCH 014/112] Implement .Unixfs.Ls() This commit was moved from ipfs/go-ipfs-http-client@eb1944fae32f4ccc43969ecd632681c2f2cbfa00 --- client/httpapi/response.go | 14 ++++++++++++- client/httpapi/unixfs.go | 43 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 27709769b..5749ca29e 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -33,7 +33,19 @@ func (r *Response) Decode(dec interface{}) error { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + n := 0 + var err error + for { + err = json.NewDecoder(r.Output).Decode(dec) + if err != nil { + break + } + n++ + } + if n > 0 && err == io.EOF { + err = nil + } + return err } type Error struct { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 21f75f85a..75e565a4a 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -11,6 +12,7 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { @@ -87,8 +89,45 @@ func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { panic("implement me") } -func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +type lsLink struct { + Name, Hash string + Size uint64 + Type unixfspb.Data_DataType +} + +type lsObject struct { + Hash string + Links []lsLink +} + +type lsOutput struct { + Objects []lsObject +} + +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out lsOutput + err := api.core().request("ls", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + + if len(out.Objects) != 1 { + return nil, errors.New("unexpected objects len") + } + + links := make([]*format.Link, len(out.Objects[0].Links)) + for i, l := range out.Objects[0].Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + links[i] = &format.Link{ + Name: l.Name, + Size: l.Size, + Cid: c, + } + } + return links, nil } func (api *UnixfsAPI) core() *HttpApi { From a6636aac599a1a8a4802ccadb885909f0ab88629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:18:21 +0100 Subject: [PATCH 015/112] Imprement partian Pin API This commit was moved from ipfs/go-ipfs-http-client@dbf90eac67d95722e88056ca16fa769a32dcd48f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 16 +++++++- client/httpapi/pin.go | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pin.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d104d98df..d5741da08 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -135,7 +135,7 @@ func (api *HttpApi) Key() iface.KeyAPI { } func (api *HttpApi) Pin() iface.PinAPI { - return nil + return (*PinAPI)(api) } func (api *HttpApi) Object() iface.ObjectAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 02c7830bb..5fdf18352 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -39,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - c := cli.NewCli() + c := cli.NewCli() //TODO: is there a better way? initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} if err := c.Run(initArgs); err != nil { @@ -100,6 +101,19 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) }, } apis[i] = NewApiWithClient(a, c) + + // node cleanup + // TODO: pass --empty-repo somehow (how?) + pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + for _, pin := range pins { //TODO: parallel + if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { + return nil, err + } + } + } return apis, nil diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go new file mode 100644 index 000000000..02dcc4222 --- /dev/null +++ b/client/httpapi/pin.go @@ -0,0 +1,78 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type PinAPI HttpApi + +type pinRefKeyObject struct { + Type string +} + +type pinRefKeyList struct { + Keys map[string]pinRefKeyObject +} + +type pin struct { + path iface.ResolvedPath + typ string +} + +func (p *pin) Path() iface.ResolvedPath { + return p.path +} + +func (p *pin) Type() string { + return p.typ +} + + +func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { + panic("implement me") +} + +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { + options, err := caopts.PinLsOptions(opts...) + if err != nil { + return nil, err + } + + var out pinRefKeyList + err = api.core().request("pin/ls"). + Option("type", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + pins := make([]iface.Pin, 0, len(out.Keys)) + for hash, p := range out.Keys { + c, err := cid.Parse(hash) + if err != nil { + return nil, err + } + pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + } + + return pins, nil +} + +func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { + return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +} + +func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { + panic("implement me") +} + +func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { + panic("implement me") +} + +func (api *PinAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 0f7c83956b2d769c801c270fe01d4819cb070881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:20:04 +0100 Subject: [PATCH 016/112] Import missing unixfs dep This commit was moved from ipfs/go-ipfs-http-client@6169321d1df6495cbd1c06cca9f2e0777a00f638 --- client/httpapi/unixfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 75e565a4a..5a1495c24 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -11,8 +11,8 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" - unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { From af197cb6d90da1b51525e68f97a30ebfc322e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 14:46:52 +0100 Subject: [PATCH 017/112] api.WithOption This commit was moved from ipfs/go-ipfs-http-client@634b00bf1a42dc7f2a2dc75f85c77e5bb5ef727c --- client/httpapi/api.go | 21 ++++++++++++++++++--- client/httpapi/api_test.go | 4 ++-- client/httpapi/requestbuilder.go | 2 ++ client/httpapi/response.go | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d5741da08..3df45f38c 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -27,6 +27,8 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string httpcli *gohttp.Client + + applyGlobal func(*RequestBuilder) } //TODO: Return errors here @@ -99,11 +101,24 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return &HttpApi{ url: url, httpcli: c, + applyGlobal: func(*RequestBuilder) {}, } } -func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { - return nil, ErrNotImplemented +func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { + options, err := caopts.ApiOptions(opts...) + if err != nil { + return nil, err + } + + subApi := *api + subApi.applyGlobal = func(req *RequestBuilder) { + if options.Offline { + req.Option("offline", options.Offline) + } + } + + return &subApi, nil } func (api *HttpApi) request(command string, args ...string) *RequestBuilder { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5fdf18352..67a2dbdce 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -104,7 +104,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) // node cleanup // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 6e5a89ebd..831e6d71c 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -86,6 +86,8 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + r.shell.applyGlobal(r) + req := NewRequest(ctx, r.shell.url, r.command, r.args...) req.Opts = r.opts req.Headers = r.headers diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 5749ca29e..f6e7f3ab7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -21,7 +21,7 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { // always drain output (response body) - ioutil.ReadAll(r.Output) + //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data return r.Output.Close() } return nil From 6d9dea62825012678ab0f117ec3cfe28c96fdaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 17:47:22 +0100 Subject: [PATCH 018/112] Implement Unixfs.Get() This commit was moved from ipfs/go-ipfs-http-client@b31bee083d1fc5002779d755b6a624b517d2b10c --- client/httpapi/apifile.go | 231 ++++++++++++++++++++++++++++++++++++++ client/httpapi/unixfs.go | 4 - 2 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/apifile.go diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go new file mode 100644 index 000000000..541189b60 --- /dev/null +++ b/client/httpapi/apifile.go @@ -0,0 +1,231 @@ +package httpapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + "github.com/ipfs/go-ipfs-files" + unixfspb "github.com/ipfs/go-unixfs/pb" +) + +func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { + if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS + var err error + p, err = api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + } + + var stat struct{ + Hash string + Type string + Size int64 // unixfs size + } + err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + if err != nil { + return nil, err + } + + switch stat.Type { + case "file": + return api.getFile(ctx, p, stat.Size) + case "directory": + return api.getDir(ctx, p, stat.Size) + default: + return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) + } +} + +type apiFile struct { + ctx context.Context + core *HttpApi + size int64 + path iface.Path + + r io.ReadCloser + at int64 +} + +func (f *apiFile) reset() error { + if f.r != nil { + f.r.Close() + } + req := f.core.request("cat", f.path.String()) + if f.at != 0 { + req.Option("offset", f.at) + } + resp, err := req.Send(f.ctx) + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + f.r = resp.Output + return nil +} + +func (f *apiFile) Read(p []byte) (int, error) { + n, err := f.r.Read(p) + if n > 0 { + f.at += int64(n) + } + return n, err +} + +func (f *apiFile) Seek(offset int64, whence int) (int64, error) { + panic("implement me") //TODO +} + +func (f *apiFile) Close() error { + if f.r != nil { + return f.r.Close() + } + return nil +} + +func (f *apiFile) Size() (int64, error) { + return f.size, nil +} + +func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + f := &apiFile{ + ctx: ctx, + core: api.core(), + size: size, + path: p, + } + + return f, f.reset() +} + +type apiIter struct { + ctx context.Context + core *UnixfsAPI + + err error + + dec *json.Decoder + curFile files.Node + cur lsLink +} + +func (it *apiIter) Err() error { + return it.err +} + +func (it *apiIter) Name() string { + return it.cur.Name +} + +func (it *apiIter) Next() bool { + var out lsOutput + if err := it.dec.Decode(&out); err != nil { + if err != io.EOF { + it.err = err + } + return false + } + + if len(out.Objects) != 1 { + it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + return false + } + + if len(out.Objects[0].Links) != 1 { + it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + return false + } + + it.cur = out.Objects[0].Links[0] + c, err := cid.Parse(it.cur.Hash) + if err != nil { + it.err = err + return false + } + + switch it.cur.Type { + case unixfspb.Data_HAMTShard: + fallthrough + case unixfspb.Data_Metadata: + fallthrough + case unixfspb.Data_Directory: + it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + case unixfspb.Data_File: + it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + default: + it.err = fmt.Errorf("file type %d not supported", it.cur.Type) + return false + } + return true +} + +func (it *apiIter) Node() files.Node { + return it.curFile +} + +type apiDir struct { + ctx context.Context + core *UnixfsAPI + size int64 + path iface.Path + + dec *json.Decoder +} + +func (d *apiDir) Close() error { + return nil +} + +func (d *apiDir) Size() (int64, error) { + return d.size, nil +} + +func (d *apiDir) Entries() files.DirIterator { + return &apiIter{ + ctx: d.ctx, + core: d.core, + dec: d.dec, + } +} + +func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + resp, err := api.core().request("ls", p.String()). + Option("resolve-size", true). + Option("stream", true).Send(ctx) + + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + d := &apiDir{ + ctx: ctx, + core: api, + size: size, + path: p, + + dec: json.NewDecoder(resp.Output), + } + + return d, nil +} + +var _ files.File = &apiFile{} +var _ files.Directory = &apiDir{} diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5a1495c24..77daf7b9b 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -85,10 +85,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return iface.IpfsPath(c), nil } -func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { - panic("implement me") -} - type lsLink struct { Name, Hash string Size uint64 From 00597e6dff81846a2dbf742dd1ab9bf546b856e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:04:22 +0100 Subject: [PATCH 019/112] Dag.Put This commit was moved from ipfs/go-ipfs-http-client@3217104469020f0c66d8457842f8221b64289d74 --- client/httpapi/api.go | 2 +- client/httpapi/dag.go | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dag.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3df45f38c..c2ec1ad67 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { } func (api *HttpApi) Dag() iface.DagAPI { - return nil + return (*DagAPI)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go new file mode 100644 index 000000000..c3b29e7ee --- /dev/null +++ b/client/httpapi/dag.go @@ -0,0 +1,69 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type DagAPI HttpApi + +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { + options, err := caopts.DagPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + codec, ok := cid.CodecToStr[options.Codec] + if !ok { + return nil, fmt.Errorf("unknowm codec %d", options.MhType) + } + + if options.MhLength != -1 { + return nil, fmt.Errorf("setting hash len is not supported yet") + } + + var out struct{ + Cid cid.Cid + } + err = api.core().request("dht/put"). + Option("hash", mht). + Option("format", codec). + Option("input-enc", options.InputEnc). + FileBody(src). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + return iface.IpldPath(out.Cid), nil +} + +func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { + panic("implement me") +} + +func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { + panic("implement me") +} + +func (api *DagAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 101910f04ed0575be19efaacb44b1bce73717033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:22:10 +0100 Subject: [PATCH 020/112] Object.New This commit was moved from ipfs/go-ipfs-http-client@60321ed42fd8f42e964e37ab5e96839fb8fd77b6 --- client/httpapi/api.go | 2 +- client/httpapi/ipldnode.go | 8 ++++ client/httpapi/object.go | 82 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/object.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c2ec1ad67..cf9a633bc 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -154,7 +154,7 @@ func (api *HttpApi) Pin() iface.PinAPI { } func (api *HttpApi) Object() iface.ObjectAPI { - return nil + return (*ObjectAPI)(api) } func (api *HttpApi) Dht() iface.DhtAPI { diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index b8e6fba01..1f584ac07 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -20,6 +20,14 @@ type ipldNode struct { api *HttpApi } +func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { + return &ipldNode{ + ctx: ctx, + path: p, + api: a, + } +} + func (n *ipldNode) RawData() []byte { r, err := n.api.Block().Get(n.ctx, n.path) if err != nil { diff --git a/client/httpapi/object.go b/client/httpapi/object.go new file mode 100644 index 000000000..9c1063af9 --- /dev/null +++ b/client/httpapi/object.go @@ -0,0 +1,82 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" +) + +type ObjectAPI HttpApi + +type objectOut struct { + Hash string +} + +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { + options, err := caopts.ObjectNewOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/new", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil +} + +func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { + panic("implement me") +} + +func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { + panic("implement me") +} + +func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { + panic("implement me") +} + +func (api *ObjectAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 31a4c3754b87e68f60525533ba9c6bb9c0b1021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:47:32 +0100 Subject: [PATCH 021/112] Fix Dag.Put This commit was moved from ipfs/go-ipfs-http-client@5b2c99abdedaf464f061511bc7b24b1ade66546f --- client/httpapi/dag.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index c3b29e7ee..152d2c6b0 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -3,30 +3,25 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-cid" "io" + "math" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) type DagAPI HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { options, err := caopts.DagPutOptions(opts...) if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - codec, ok := cid.CodecToStr[options.Codec] if !ok { return nil, fmt.Errorf("unknowm codec %d", options.MhType) @@ -39,12 +34,19 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPu var out struct{ Cid cid.Cid } - err = api.core().request("dht/put"). - Option("hash", mht). + req := api.core().request("dag/put"). Option("format", codec). - Option("input-enc", options.InputEnc). - FileBody(src). - Exec(ctx, &out) + Option("input-enc", options.InputEnc) + + if options.MhType != math.MaxUint64 { + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + req.Option("hash", mht) + } + + err = req.FileBody(src).Exec(ctx, &out) if err != nil { return nil, err } @@ -56,7 +58,7 @@ func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error panic("implement me") } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { panic("implement me") } From c213e2654235ffbdb6ce2f3a7df3c027ba53e02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 21:55:45 +0100 Subject: [PATCH 022/112] Unixfs.Add progress events This commit was moved from ipfs/go-ipfs-http-client@2f3a77b686ee8c4194ee6c21cbdecd57f2081519 --- client/httpapi/api.go | 4 +-- client/httpapi/api_test.go | 6 ++--- client/httpapi/apifile.go | 24 +++++++++--------- client/httpapi/block.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/ipldnode.go | 4 +-- client/httpapi/key.go | 5 ++-- client/httpapi/object.go | 2 +- client/httpapi/pin.go | 3 +-- client/httpapi/unixfs.go | 51 ++++++++++++++++++++++++++++++++++---- 10 files changed, 71 insertions(+), 32 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cf9a633bc..c686a8d0d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -99,8 +99,8 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } return &HttpApi{ - url: url, - httpcli: c, + url: url, + httpcli: c, applyGlobal: func(*RequestBuilder) {}, } } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 67a2dbdce..b822378f7 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -53,7 +53,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} if err := c.Run(connectArgs); err != nil { return nil, err } @@ -95,8 +95,8 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := &gohttp.Client{ Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, DisableCompression: true, }, } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 541189b60..d9da23975 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -22,12 +22,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } } - var stat struct{ + var stat struct { Hash string Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -43,12 +43,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } type apiFile struct { - ctx context.Context + ctx context.Context core *HttpApi size int64 path iface.Path - r io.ReadCloser + r io.ReadCloser at int64 } @@ -96,7 +96,7 @@ func (f *apiFile) Size() (int64, error) { func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { f := &apiFile{ - ctx: ctx, + ctx: ctx, core: api.core(), size: size, path: p, @@ -106,14 +106,14 @@ func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (fi } type apiIter struct { - ctx context.Context + ctx context.Context core *UnixfsAPI err error - dec *json.Decoder + dec *json.Decoder curFile files.Node - cur lsLink + cur lsLink } func (it *apiIter) Err() error { @@ -179,7 +179,7 @@ func (it *apiIter) Node() files.Node { } type apiDir struct { - ctx context.Context + ctx context.Context core *UnixfsAPI size int64 path iface.Path @@ -197,9 +197,9 @@ func (d *apiDir) Size() (int64, error) { func (d *apiDir) Entries() files.DirIterator { return &apiIter{ - ctx: d.ctx, + ctx: d.ctx, core: d.core, - dec: d.dec, + dec: d.dec, } } @@ -216,7 +216,7 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (fil } d := &apiDir{ - ctx: ctx, + ctx: ctx, core: api, size: size, path: p, diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 185aa0a42..f933d3c4a 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -16,7 +16,7 @@ import ( type BlockAPI HttpApi type blockStat struct { - Key string + Key string BSize int `json:"Size"` } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 152d2c6b0..20c233196 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -31,7 +31,7 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPut return nil, fmt.Errorf("setting hash len is not supported yet") } - var out struct{ + var out struct { Cid cid.Cid } req := api.core().request("dag/put"). diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 1f584ac07..8e028e1a7 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -22,9 +22,9 @@ type ipldNode struct { func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ - ctx: ctx, + ctx: ctx, path: p, - api: a, + api: a, } } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 87b573f98..3653526eb 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -13,7 +13,7 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` - Id string + Id string } func (k *keyOutput) Name() string { @@ -35,7 +35,6 @@ func (k *keyOutput) valid() error { return err } - func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { options, err := caopts.KeyGenerateOptions(opts...) if err != nil { @@ -65,7 +64,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { - var id struct{ID string} + var id struct{ ID string } if err := api.core().request("id").Exec(ctx, &id); err != nil { return nil, err } diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 9c1063af9..6259473c7 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,7 +14,7 @@ import ( type ObjectAPI HttpApi type objectOut struct { - Hash string + Hash string } func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 02dcc4222..ea34dc540 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -20,7 +20,7 @@ type pinRefKeyList struct { type pin struct { path iface.ResolvedPath - typ string + typ string } func (p *pin) Path() iface.ResolvedPath { @@ -31,7 +31,6 @@ func (p *pin) Type() string { return p.typ } - func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { panic("implement me") } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 77daf7b9b..7e63d241f 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -2,9 +2,11 @@ package httpapi import ( "context" + "encoding/json" "fmt" "github.com/ipfs/go-cid" "github.com/pkg/errors" + "io" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -39,7 +41,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). - //Option("", options.Events). Option("fscache", options.FsCache). Option("hidden", options.Hidden). Option("inline", options.Inline). @@ -47,11 +48,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). - //Option("", options.Progress). Option("silent", options.Silent). Option("stdin-name", options.StdinName). Option("wrap-with-directory", options.Wrap). - Option("quieter", true) // TODO: rm after event impl + Option("progress", options.Progress) if options.RawLeavesSet { req.Option("raw-leaves", options.RawLeaves) @@ -73,9 +73,50 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix } var out addEvent - if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + resp, err := req.Send(ctx) + if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Output.Close() + dec := json.NewDecoder(resp.Output) +loop: + for { + var evt addEvent + switch err := dec.Decode(&evt); err { + case nil: + case io.EOF: + break loop + default: + return nil, err + } + out = evt + + if options.Events != nil { + ifevt := &iface.AddEvent{ + Name: out.Name, + Size: out.Size, + Bytes: out.Bytes, + } + + if out.Hash != "" { + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + ifevt.Path = iface.IpfsPath(c) + } + + select { + case options.Events <- ifevt: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } c, err := cid.Parse(out.Hash) if err != nil { @@ -120,7 +161,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, err links[i] = &format.Link{ Name: l.Name, Size: l.Size, - Cid: c, + Cid: c, } } return links, nil From 266c2f92c50df9b50bbc5aa093c793bb5d1a2265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 14:33:03 +0100 Subject: [PATCH 023/112] Implement Key API This commit was moved from ipfs/go-ipfs-http-client@281b2bf65b19b17e4f0c047ac8f9901306bf171f --- client/httpapi/key.go | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 3653526eb..417415ddb 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -49,18 +50,47 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { - panic("implement me") + options, err := caopts.KeyRenameOptions(opts...) + if err != nil { + return nil, false, err + } + + var out struct{ + Was string + Now string + Id string + Overwrite bool + } + err = api.core().request("key/rename", oldName, newName). + Option("force", options.Force). + Exec(ctx, &out) + if err != nil { + return nil, false, err + } + + id := &keyOutput{JName: out.Now, Id: out.Id} + return id, out.Overwrite, id.valid() } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { - panic("implement me") + var out struct{ Keys []*keyOutput } + if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]iface.Key, len(out.Keys)) + for i, k := range out.Keys { + if err := k.valid(); err != nil { + return nil, err + } + res[i] = k + } + + return res, nil } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { @@ -70,14 +100,19 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { } out := keyOutput{JName: "self", Id: id.ID} - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { - panic("implement me") + var out struct{ Keys []keyOutput } + if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + return nil, err + } + if len(out.Keys) != 1 { + return nil, errors.New("got unexpected number of keys back") + } + + return &out.Keys[0], out.Keys[0].valid() } func (api *KeyAPI) core() *HttpApi { From 9d647d011efa551bae28c50cafb402a35d33b04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 15:02:15 +0100 Subject: [PATCH 024/112] Implement Pin API This commit was moved from ipfs/go-ipfs-http-client@5bb7a581323aad4dd765a0916f6552cd5902d1e2 --- client/httpapi/pin.go | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ea34dc540..34cafad71 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -31,8 +33,14 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { - panic("implement me") +func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { + options, err := caopts.PinAddOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/add", p.String()). + Option("recursive", options.Recursive).Exec(ctx, nil) } func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { @@ -65,11 +73,85 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { - panic("implement me") + options, err := caopts.PinUpdateOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/update"). + Option("unpin", options.Unpin).Exec(ctx, nil) } -func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { - panic("implement me") +type pinVerifyRes struct { + Cid string + JOk bool `json:"Ok"` + JBadNodes []*badNode `json:"BadNodes,omitempty"` +} + +func (r *pinVerifyRes) Ok() bool { + return r.JOk +} + +func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { + out := make([]iface.BadPinNode, len(r.JBadNodes)) + for i, n := range r.JBadNodes { + out[i] = n + } + return out +} + +type badNode struct { + Cid string + JErr string `json:"Err"` +} + +func (n *badNode) Path() iface.ResolvedPath { + c, err := cid.Parse(n.Cid) + if err != nil { + return nil // todo: handle this better + } + return iface.IpldPath(c) +} + +func (n *badNode) Err() error { + if n.JErr != "" { + return errors.New(n.JErr) + } + if _, err := cid.Parse(n.Cid); err != nil { + return err + } + return nil +} + +func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { + resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan iface.PinStatus) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + for { + var out pinVerifyRes + if err := dec.Decode(&out); err != nil { + return // todo: handle non io.EOF somehow + } + + select { + case res <- &out: + case <-ctx.Done(): + return + } + } + }() + + return res, nil } func (api *PinAPI) core() *HttpApi { From eaa19388d4d86eebf09d93a3a637e0690df722bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 16:31:07 +0100 Subject: [PATCH 025/112] Implement Object API This commit was moved from ipfs/go-ipfs-http-client@38149e46c8c3a1544dc3e3108ef72ffbc885e410 --- client/httpapi/block.go | 2 +- client/httpapi/ipldnode.go | 7 +- client/httpapi/key.go | 8 +- client/httpapi/object.go | 216 +++++++++++++++++++++++++++++++++---- client/httpapi/pin.go | 6 +- 5 files changed, 208 insertions(+), 31 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f933d3c4a..f8334a109 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -67,7 +67,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + resp, err := api.core().request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 8e028e1a7..2294a95b4 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "errors" - "fmt" "io/ioutil" "strconv" @@ -20,11 +19,11 @@ type ipldNode struct { api *HttpApi } -func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { +func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ ctx: ctx, path: p, - api: a, + api: api, } } @@ -47,7 +46,7 @@ func (n *ipldNode) Cid() cid.Cid { } func (n *ipldNode) String() string { - return fmt.Sprintf("[Block %s]", n.Cid()) + return n.Cid().String() } func (n *ipldNode) Loggable() map[string]interface{} { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 417415ddb..5087b6d99 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -59,10 +59,10 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o return nil, false, err } - var out struct{ - Was string - Now string - Id string + var out struct { + Was string + Now string + Id string Overwrite bool } err = api.core().request("key/rename", oldName, newName). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6259473c7..4bc787991 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -1,14 +1,17 @@ package httpapi import ( + "bytes" "context" - "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" ) type ObjectAPI HttpApi @@ -37,44 +40,219 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil } -func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + options, err := caopts.ObjectPutOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/put"). + Option("inputenc", options.InputEnc). + Option("datafieldenc", options.DataType). + Option("pin", options.Pin). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { - panic("implement me") +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { + r, err := api.core().Block().Get(ctx, p) + if err != nil { + return nil, err + } + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { - panic("implement me") +func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("object/data", p.String()).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + //TODO: make Data return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } + + return b, nil } -func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out struct { + Links []struct { + Name string + Hash string + Size uint64 + } + } + if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]*format.Link, len(out.Links)) + for i, l := range out.Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + + res[i] = &format.Link{ + Cid: c, + Name: l.Name, + Size: l.Size, + } + } + + return res, nil } -func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { - panic("implement me") +func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { + var out struct { + Hash string + NumLinks int + BlockSize int + LinksSize int + DataSize int + CumulativeSize int + } + if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return &iface.ObjectStat{ + Cid: c, + NumLinks: out.NumLinks, + BlockSize: out.BlockSize, + LinksSize: out.LinksSize, + DataSize: out.DataSize, + CumulativeSize: out.CumulativeSize, + }, nil } func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { - panic("implement me") + options, err := caopts.ObjectAddLinkOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + Option("create", options.Create). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { - panic("implement me") + var out objectOut + err := api.core().request("object/patch/rm-link", base.String(), link). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/append-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/set-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { - panic("implement me") +type change struct { + Type iface.ChangeType + Path string + Before cid.Cid + After cid.Cid +} + +func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { + var out struct { + Changes []change + } + if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]iface.ObjectChange, len(out.Changes)) + for i, ch := range out.Changes { + res[i] = iface.ObjectChange{ + Type: ch.Type, + Path: ch.Path, + } + if ch.Before != cid.Undef { + res[i].Before = iface.IpfsPath(ch.Before) + } + if ch.After != cid.Undef { + res[i].After = iface.IpfsPath(ch.After) + } + } + return res, nil } func (api *ObjectAPI) core() *HttpApi { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 34cafad71..1624b7867 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -83,8 +83,8 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` + Cid string + JOk bool `json:"Ok"` JBadNodes []*badNode `json:"BadNodes,omitempty"` } @@ -101,7 +101,7 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { } type badNode struct { - Cid string + Cid string JErr string `json:"Err"` } From ed9f2dd091693e914e9de072b2c23c2f991210a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:01:33 +0100 Subject: [PATCH 026/112] Implement DHT Api This commit was moved from ipfs/go-ipfs-http-client@c77355067a22300a8abe6b93b354de0ae0a3b548 --- client/httpapi/api.go | 2 +- client/httpapi/dht.go | 98 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dht.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c686a8d0d..208b958e9 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -158,7 +158,7 @@ func (api *HttpApi) Object() iface.ObjectAPI { } func (api *HttpApi) Dht() iface.DhtAPI { - return nil + return (*DhtAPI)(api) } func (api *HttpApi) Swarm() iface.SwarmAPI { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go new file mode 100644 index 000000000..17d3b0500 --- /dev/null +++ b/client/httpapi/dht.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "context" + "encoding/json" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + notif "github.com/libp2p/go-libp2p-routing/notifications" +) + +type DhtAPI HttpApi + +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + if err != nil { + return peerstore.PeerInfo{}, err + } + if resp.Error != nil { + return peerstore.PeerInfo{}, resp.Error + } + defer resp.Close() + dec := json.NewDecoder(resp.Output) + for { + if err := dec.Decode(&out); err != nil { + return peerstore.PeerInfo{}, err + } + if out.Type == notif.FinalPeer { + return out.Responses[0], nil + } + } +} + +func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { + options, err := caopts.DhtFindProvidersOptions(opts...) + if err != nil { + return nil, err + } + resp, err := api.core().request("dht/findprovs", p.String()). + Option("num-providers", options.NumProviders). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan peerstore.PeerInfo) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + + for { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + + if err := dec.Decode(&out); err != nil { + return // todo: handle this somehow + } + if out.Type == notif.Provider { + for _, pi := range out.Responses { + select { + case res <- pi: + case <-ctx.Done(): + return + } + } + } + } + }() + + return res, nil +} + +func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { + options, err := caopts.DhtProvideOptions(opts...) + if err != nil { + return err + } + + return api.core().request("dht/provide", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) +} + +func (api *DhtAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cae0ff2379f29a205426f7a8199c398885f33897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:36:35 +0100 Subject: [PATCH 027/112] Implement Swarm Api This commit was moved from ipfs/go-ipfs-http-client@01105690d2da6e3f3581fa364c6284248a251a35 --- client/httpapi/api.go | 2 +- client/httpapi/swarm.go | 176 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/swarm.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 208b958e9..a5b56eff6 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -162,7 +162,7 @@ func (api *HttpApi) Dht() iface.DhtAPI { } func (api *HttpApi) Swarm() iface.SwarmAPI { - return nil + return (*SwarmAPI)(api) } func (api *HttpApi) PubSub() iface.PubSubAPI { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go new file mode 100644 index 000000000..cf0ee2a9b --- /dev/null +++ b/client/httpapi/swarm.go @@ -0,0 +1,176 @@ +package httpapi + +import ( + "context" + "github.com/libp2p/go-libp2p-protocol" + "time" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + "github.com/multiformats/go-multiaddr" +) + +type SwarmAPI HttpApi + +func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + saddrs := make([]string, len(pi.Addrs)) + for i, addr := range pi.Addrs { + saddrs[i] = addr.String() + } + + return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) +} + +func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { + return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) +} + +type streamInfo struct { + Protocol string +} + +type connInfo struct { + Addr string + Peer string + JLatency time.Duration `json:"Latency"` + Muxer string + JDirection inet.Direction `json:"Direction"` + JStreams []streamInfo `json:"Streams"` +} + +func (c *connInfo) valid() error { + _, err := multiaddr.NewMultiaddr(c.Addr) + if err != nil { + return err + } + + _, err = peer.IDB58Decode(c.Peer) + return err +} + +func (c *connInfo) ID() peer.ID { + id, _ := peer.IDB58Decode(c.Peer) + return id +} + +func (c *connInfo) Address() multiaddr.Multiaddr { + a, _ := multiaddr.NewMultiaddr(c.Addr) + return a +} + +func (c *connInfo) Direction() inet.Direction { + return c.JDirection +} + +func (c *connInfo) Latency() (time.Duration, error) { + return c.JLatency, nil +} + +func (c *connInfo) Streams() ([]protocol.ID, error) { + res := make([]protocol.ID, len(c.JStreams)) + for i, stream := range c.JStreams { + res[i] = protocol.ID(stream.Protocol) + } + return res, nil +} + +func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { + var out struct { + Peers []*connInfo + } + + err := api.core().request("swarm/peers"). + Option("streams", true). + Option("latency", true). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + res := make([]iface.ConnectionInfo, len(out.Peers)) + for i, conn := range out.Peers { + if err := conn.valid(); err != nil { + return nil, err + } + res[i] = conn + } + + return res, nil +} + +func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) { + var out struct { + Addrs map[string][]string + } + if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + return nil, err + } + res := map[peer.ID][]multiaddr.Multiaddr{} + for spid, saddrs := range out.Addrs { + addrs := make([]multiaddr.Multiaddr, len(saddrs)) + + for i, addr := range saddrs { + a, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + addrs[i] = a + } + + pid, err := peer.IDB58Decode(spid) + if err != nil { + return nil, err + } + + res[pid] = addrs + } + + return res, nil +} + +func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c2e0872f6dfba6f6d3f74a4b9eb755053a3f483e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:19:50 +0100 Subject: [PATCH 028/112] Implement PubSub Api This commit was moved from ipfs/go-ipfs-http-client@7abddda1d33595dffb0e0da782b52660147f6855 --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 2 +- client/httpapi/pubsub.go | 129 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pubsub.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index a5b56eff6..3a8e0e003 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -166,5 +166,5 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { } func (api *HttpApi) PubSub() iface.PubSubAPI { - return nil + return (*PubsubAPI)(api) } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b822378f7..1acece88c 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,7 +47,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go new file mode 100644 index 000000000..3c5e3c50d --- /dev/null +++ b/client/httpapi/pubsub.go @@ -0,0 +1,129 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type PubsubAPI HttpApi + +func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + return nil, err + } + + return out.Strings, nil +} + +func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { + options, err := caopts.PubSubPeersOptions(opts...) + if err != nil { + return nil, err + } + + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]peer.ID, len(out.Strings)) + for i, sid := range out.Strings { + id, err := peer.IDB58Decode(sid) + if err != nil { + return nil, err + } + res[i] = id + } + return res, nil +} + +func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { + return api.core().request("pubsub/pub", topic). + FileBody(bytes.NewReader(message)). + Exec(ctx, nil) +} + +type pubsubSub struct { + io.Closer + dec *json.Decoder +} + +type pubsubMessage struct { + JFrom []byte `json:"from,omitempty"` + JData []byte `json:"data,omitempty"` + JSeqno []byte `json:"seqno,omitempty"` + JTopicIDs []string `json:"topicIDs,omitempty"` +} + +func (msg *pubsubMessage) valid() error { + _, err := peer.IDFromBytes(msg.JFrom) + return err +} + +func (msg *pubsubMessage) From() peer.ID { + id, _ := peer.IDFromBytes(msg.JFrom) + return id +} + +func (msg *pubsubMessage) Data() []byte { + return msg.JData +} + +func (msg *pubsubMessage) Seq() []byte { + return msg.JSeqno +} + +func (msg *pubsubMessage) Topics() []string { + return msg.JTopicIDs +} + +func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { + // TODO: handle ctx + + var msg pubsubMessage + if err := s.dec.Decode(&msg); err != nil { + return nil, err + } + return &msg, msg.valid() +} + +func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + options, err := caopts.PubSubSubscribeOptions(opts...) + if err != nil { + return nil, err + } + + resp, err := api.core().request("pubsub/sub", topic). + Option("discover", options.Discover). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return &pubsubSub{ + Closer: resp, + dec: json.NewDecoder(resp.Output), + }, nil +} + +func (api *PubsubAPI) core() *HttpApi { + return (*HttpApi)(api) +} + From 9b24cf0aafa94a7b4cb4401b6fbb9c959bde09b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:49:11 +0100 Subject: [PATCH 029/112] Use cids in DHT calls This commit was moved from ipfs/go-ipfs-http-client@163b25f8b88eb98ddae50978440e3940070fabc0 --- client/httpapi/dht.go | 20 ++++++++++++++++++-- client/httpapi/pubsub.go | 3 +-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 17d3b0500..8fedabbf8 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -42,7 +42,13 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err != nil { return nil, err } - resp, err := api.core().request("dht/findprovs", p.String()). + + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + resp, err := api.core().request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -60,6 +66,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop for { var out struct { + Extra string Type notif.QueryEventType Responses []peerstore.PeerInfo } @@ -67,6 +74,10 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } + if out.Type == notif.QueryError { + return // usually a 'not found' error + // todo: handle other errors + } if out.Type == notif.Provider { for _, pi := range out.Responses { select { @@ -88,7 +99,12 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", p.String()). + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return err + } + + return api.core().request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3c5e3c50d..fb9bb7460 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -119,11 +119,10 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return &pubsubSub{ Closer: resp, - dec: json.NewDecoder(resp.Output), + dec: json.NewDecoder(resp.Output), }, nil } func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } - From b7e258cc1015e518198c50b02fcc6ebdd8767375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Jan 2019 21:31:09 +0100 Subject: [PATCH 030/112] Reimplement DAG as DAGService This commit was moved from ipfs/go-ipfs-http-client@f34a5f6d2569bb21f022d3231e811b3db6262ca8 --- client/httpapi/api.go | 6 +- client/httpapi/apifile.go | 5 ++ client/httpapi/dag.go | 115 ++++++++++++++++++++++++-------------- 3 files changed, 83 insertions(+), 43 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3a8e0e003..7d4fbd0e0 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -10,6 +10,8 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -137,8 +139,8 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() iface.DagAPI { - return (*DagAPI)(api) +func (api *HttpApi) Dag() format.DAGService { + return (*HttpDagServ)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index d9da23975..99ae72059 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -125,6 +125,11 @@ func (it *apiIter) Name() string { } func (it *apiIter) Next() bool { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + var out lsOutput if err := it.dec.Decode(&out); err != nil { if err != io.EOF { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 20c233196..de2934c2f 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -1,71 +1,104 @@ package httpapi import ( + "bytes" "context" "fmt" - "io" - "math" + "io/ioutil" + "sync" "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" ) -type DagAPI HttpApi +type HttpDagServ HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { - options, err := caopts.DagPutOptions(opts...) +func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { + r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) if err != nil { return nil, err } - codec, ok := cid.CodecToStr[options.Codec] - if !ok { - return nil, fmt.Errorf("unknowm codec %d", options.MhType) + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err } - if options.MhLength != -1 { - return nil, fmt.Errorf("setting hash len is not supported yet") + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, err } - var out struct { - Cid cid.Cid - } - req := api.core().request("dag/put"). - Option("format", codec). - Option("input-enc", options.InputEnc) + return format.DefaultBlockDecoder.Decode(blk) +} - if options.MhType != math.MaxUint64 { - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) +func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { + out := make(chan *format.NodeOption) + wg := sync.WaitGroup{} + wg.Add(len(cids)) + + for _, c := range cids { + // TODO: Consider limiting concurrency of this somehow + go func() { + defer wg.Done() + n, err := api.Get(ctx, c) + + select { + case out <- &format.NodeOption{Node: n, Err: err}: + case <-ctx.Done(): + } + }() + } + return out +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + c := nd.Cid() + prefix := c.Prefix() + format := cid.CodecToStr[prefix.Codec] + if prefix.Version == 0 { + format = "v0" + } + + stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), + options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) + if err != nil { + return err + } + if !stat.Path().Cid().Equals(c) { + return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String()) + } + return nil +} + +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + for _, nd := range nds { + // TODO: optimize + if err := api.Add(ctx, nd); err != nil { + return err } - req.Option("hash", mht) } + return nil +} - err = req.FileBody(src).Exec(ctx, &out) - if err != nil { - return nil, err +func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { + return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? +} + +func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { + for _, c := range cids { + // TODO: optimize + if err := api.Remove(ctx, c); err != nil { + return err + } } - - return iface.IpldPath(out.Cid), nil + return nil } -func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { - panic("implement me") -} - -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { - panic("implement me") -} - -func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { - panic("implement me") -} - -func (api *DagAPI) core() *HttpApi { +func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From c3f2970f8a7302344154e6d7fe74485f0dd272c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:42:32 +0100 Subject: [PATCH 031/112] block: Pin option This commit was moved from ipfs/go-ipfs-http-client@def66919dfbee162595852334468719f57158121 --- client/httpapi/block.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f8334a109..24df9629e 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -53,6 +53,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). + Option("pin", options.Pin). FileBody(r) var out blockStat From af2edd12eb44f477ef2e0dc22ae31a75df645b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:00 +0100 Subject: [PATCH 032/112] tests: enable filestore This commit was moved from ipfs/go-ipfs-http-client@904e8eeeb1b677d458c819122d91264040120d3b --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 1acece88c..bf1d6b9b0 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,6 +47,11 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} + if err := c.Run(filestoreArgs); err != nil { + return nil, err + } + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err From 75cf2be1026a7f206e61d164342dae4807d36bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:26 +0100 Subject: [PATCH 033/112] apifile: Implement Seek This commit was moved from ipfs/go-ipfs-http-client@69cc3e8106552605ecfdfa5c6f352996dc6f9497 --- client/httpapi/apifile.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 99ae72059..ec916d8e2 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,13 +6,15 @@ import ( "fmt" "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs-files" - unixfspb "github.com/ipfs/go-unixfs/pb" ) +const forwardSeekLimit = 1 << 14 //16k + func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error @@ -80,7 +82,24 @@ func (f *apiFile) Read(p []byte) (int, error) { } func (f *apiFile) Seek(offset int64, whence int) (int64, error) { - panic("implement me") //TODO + switch whence { + case io.SeekEnd: + offset = f.size + offset + case io.SeekCurrent: + offset = f.at + offset + } + if f.at == offset { //noop + return offset, nil + } + + if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + + f.at += r + return f.at, err + } + f.at = offset + return f.at, f.reset() } func (f *apiFile) Close() error { @@ -156,17 +175,17 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case unixfspb.Data_HAMTShard: + case iface.THAMTShard: fallthrough - case unixfspb.Data_Metadata: + case iface.TMetadata: fallthrough - case unixfspb.Data_Directory: + case iface.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case unixfspb.Data_File: + case iface.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err From bb83ccb15feb4854c7bdb4821fbcf90b89f36163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:44:48 +0100 Subject: [PATCH 034/112] response: read trailing error headers This commit was moved from ipfs/go-ipfs-http-client@83dfd84ba8bf2341478bb98d437ae1f14fe29e18 --- client/httpapi/response.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f6e7f3ab7..745c9a2a9 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -2,6 +2,7 @@ package httpapi import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,24 @@ import ( files "github.com/ipfs/go-ipfs-files" ) +type trailerReader struct { + resp *http.Response +} + +func (r *trailerReader) Read(b []byte) (int, error) { + n, err := r.resp.Body.Read(b) + if err != nil { + if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + err = errors.New(e) + } + } + return n, err +} + +func (r *trailerReader) Close() error { + return r.resp.Body.Close() +} + type Response struct { Output io.ReadCloser Error *Error @@ -56,9 +75,6 @@ type Error struct { func (e *Error) Error() string { var out string - if e.Command != "" { - out = e.Command + ": " - } if e.Code != 0 { out = fmt.Sprintf("%s%d: ", out, e.Code) } @@ -93,7 +109,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.Output = resp.Body + nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ Command: r.Command, From 88139ddc50e111e39721990bb86bb1c018c488b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:45:24 +0100 Subject: [PATCH 035/112] dag: Interface updates This commit was moved from ipfs/go-ipfs-http-client@7bea2efb45cba072b4e24c20b46a4372be3a716d --- client/httpapi/api.go | 3 +-- client/httpapi/dag.go | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 7d4fbd0e0..2c7a97c99 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -139,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() format.DAGService { +func (api *HttpApi) Dag() iface.APIDagService { return (*HttpDagServ)(api) } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index de2934c2f..eacf631b9 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -15,7 +15,9 @@ import ( "github.com/ipfs/go-ipld-format" ) -type HttpDagServ HttpApi +type httpNodeAdder HttpApi +type HttpDagServ httpNodeAdder +type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) @@ -56,7 +58,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for return out } -func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { +func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() format := cid.CodecToStr[prefix.Codec] @@ -65,7 +67,9 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), - options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) + options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.Format(format), + options.Block.Pin(pin)) if err != nil { return err } @@ -75,16 +79,36 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { return nil } -func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { +func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error { for _, nd := range nds { // TODO: optimize - if err := api.Add(ctx, nd); err != nil { + if err := api.add(ctx, nd, pin); err != nil { return err } } return nil } +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, false) +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, false) +} + +func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, true) +} + +func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, true) +} + +func (api *HttpDagServ) Pinning() format.NodeAdder { + return (*pinningHttpNodeAdder)(api) +} + func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } @@ -99,6 +123,10 @@ func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { return nil } +func (api *httpNodeAdder) core() *HttpApi { + return (*HttpApi)(api) +} + func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From 93bfcf91cf979b267758c04aa14c6435a687fc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:46:00 +0100 Subject: [PATCH 036/112] unixfs: updated ls This commit was moved from ipfs/go-ipfs-http-client@bb8d9d1a60e4161c4cf3a33c468746050f267a12 --- client/httpapi/unixfs.go | 94 ++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 7e63d241f..67d05309b 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" - unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" ) @@ -129,7 +128,7 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type unixfspb.Data_DataType + Type iface.FileType } type lsObject struct { @@ -141,30 +140,87 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { - var out lsOutput - err := api.core().request("ls", p.String()).Exec(ctx, &out) +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err } - if len(out.Objects) != 1 { - return nil, errors.New("unexpected objects len") + resp, err := api.core().request("ls", p.String()). + Option("resolve-type", options.ResolveChildren). + Option("size", options.ResolveChildren). + Option("stream", true). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error } - links := make([]*format.Link, len(out.Objects[0].Links)) - for i, l := range out.Objects[0].Links { - c, err := cid.Parse(l.Hash) - if err != nil { - return nil, err + dec := json.NewDecoder(resp.Output) + out := make(chan iface.LsLink) + + go func() { + defer resp.Close() + defer close(out) + + for { + var link lsOutput + if err := dec.Decode(&link); err != nil { + if err == io.EOF { + return + } + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects[0].Links) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case <-ctx.Done(): + } + return + } + + l0 := link.Objects[0].Links[0] + + c, err := cid.Decode(l0.Hash) + if err != nil { + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + select { + case out <- iface.LsLink{ + Link: &format.Link{ + Cid: c, + Name: l0.Name, + Size: l0.Size, + }, + Size: l0.Size, + Type: l0.Type, + }: + case <-ctx.Done(): + } } - links[i] = &format.Link{ - Name: l.Name, - Size: l.Size, - Cid: c, - } - } - return links, nil + }() + + return out, nil } func (api *UnixfsAPI) core() *HttpApi { From 1bc854bf2169f82d2275232e123adae721b8dc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:53:02 +0100 Subject: [PATCH 037/112] pin: handle Rm options This commit was moved from ipfs/go-ipfs-http-client@e85e856ea2841cd3e0cf4cb7b87441539af42bbe --- client/httpapi/pin.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 1624b7867..237b32329 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -68,8 +68,15 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { - return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { + options, err := caopts.PinRmOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/rm", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { From c543354b1772323cb2cfbf41c06522297bf27667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Feb 2019 22:36:30 +0100 Subject: [PATCH 038/112] Check for redirects This commit was moved from ipfs/go-ipfs-http-client@19c65db4f0fd1549fc6f224efd868d4f00da997d --- client/httpapi/api.go | 18 +++++++++++++----- client/httpapi/requestbuilder.go | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 2c7a97c99..da0836d4a 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -2,6 +2,7 @@ package httpapi import ( "errors" + "fmt" "io/ioutil" gohttp "net/http" "os" @@ -27,7 +28,7 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string - httpcli *gohttp.Client + httpcli gohttp.Client applyGlobal func(*RequestBuilder) } @@ -50,8 +51,8 @@ func NewPathApi(p string) iface.CoreAPI { return NewApi(a) } -func ApiAddr(p string) ma.Multiaddr { - baseDir, err := homedir.Expand(p) +func ApiAddr(ipfspath string) ma.Multiaddr { + baseDir, err := homedir.Expand(ipfspath) if err != nil { return nil } @@ -99,11 +100,18 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } } - return &HttpApi{ + api := &HttpApi{ url: url, - httpcli: c, + httpcli: *c, applyGlobal: func(*RequestBuilder) {}, } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 831e6d71c..628ad03cd 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -92,7 +92,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { req.Opts = r.opts req.Headers = r.headers req.Body = r.body - return req.Send(r.shell.httpcli) + return req.Send(&r.shell.httpcli) } // Exec sends the request a request and decodes the response. From ef5bf40df1d0d790aeb213147c53412145894bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Feb 2019 18:40:38 +0100 Subject: [PATCH 039/112] Update imports to use extracted interface This commit was moved from ipfs/go-ipfs-http-client@fad467bc437a1282ed3813d828c6f8ec5b7b2d7a --- client/httpapi/api.go | 5 ++--- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 9 ++++----- client/httpapi/block.go | 4 ++-- client/httpapi/dag.go | 5 ++--- client/httpapi/dht.go | 4 ++-- client/httpapi/ipldnode.go | 3 +-- client/httpapi/key.go | 5 ++--- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 5 ++--- client/httpapi/path.go | 3 +-- client/httpapi/pin.go | 8 ++++---- client/httpapi/pubsub.go | 5 ++--- client/httpapi/swarm.go | 5 ++--- client/httpapi/unixfs.go | 11 +++++------ 15 files changed, 37 insertions(+), 48 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index da0836d4a..698e36524 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,9 +9,8 @@ import ( "path" "strings" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + iface "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index bf1d6b9b0..5621ab87f 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,10 +10,9 @@ import ( "strconv" "testing" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec916d8e2..1e5f61a9a 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/ipfs/go-cid" "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -92,8 +91,8 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { return offset, nil } - if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 24df9629e..45c73472c 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -8,8 +8,8 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index eacf631b9..d0059698a 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -7,12 +7,11 @@ import ( "io/ioutil" "sync" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/options" ) type httpNodeAdder HttpApi diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 8fedabbf8..dc7dd6bea 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -3,9 +3,9 @@ package httpapi import ( "context" "encoding/json" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 2294a95b4..43fa5d50d 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -6,11 +6,10 @@ import ( "io/ioutil" "strconv" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) type ipldNode struct { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 5087b6d99..dc2adf8b7 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,8 @@ import ( "context" "errors" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index fbc440b96..d760e6188 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -3,10 +3,10 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-ipfs/namesys/opts" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/options/namesys" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 4bc787991..3b648d82b 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,12 +6,11 @@ import ( "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 5701326fc..68dc2633d 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,11 +3,10 @@ package httpapi import ( "context" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 237b32329..4c4e5713c 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,11 +3,11 @@ package httpapi import ( "context" "encoding/json" - "github.com/ipfs/go-cid" - "github.com/pkg/errors" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-cid" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/pkg/errors" ) type PinAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index fb9bb7460..334509780 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,9 +6,8 @@ import ( "encoding/json" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index cf0ee2a9b..d179b6540 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -2,14 +2,13 @@ package httpapi import ( "context" - "github.com/libp2p/go-libp2p-protocol" "time" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/interface-go-ipfs-core" inet "github.com/libp2p/go-libp2p-net" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 67d05309b..1f340b657 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -3,16 +3,15 @@ package httpapi import ( "context" "encoding/json" + "errors" "fmt" - "github.com/ipfs/go-cid" - "github.com/pkg/errors" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -208,7 +207,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf select { case out <- iface.LsLink{ Link: &format.Link{ - Cid: c, + Cid: c, Name: l0.Name, Size: l0.Size, }, From 0813d808b54341414441a56e5cd3a7722a516a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:29:31 +0100 Subject: [PATCH 040/112] Fix govet warning in Dag This commit was moved from ipfs/go-ipfs-http-client@62552b33959401c7a9d9463dffba433e522b2f15 --- client/httpapi/dag.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index d0059698a..3f54ced34 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -44,7 +44,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for for _, c := range cids { // TODO: Consider limiting concurrency of this somehow - go func() { + go func(c cid.Cid) { defer wg.Done() n, err := api.Get(ctx, c) @@ -52,7 +52,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for case out <- &format.NodeOption{Node: n, Err: err}: case <-ctx.Done(): } - }() + }(c) } return out } From abc30e384fafe157490df0dae80a22ddeb6255a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:48:04 +0100 Subject: [PATCH 041/112] Register iptb plugin once This commit was moved from ipfs/go-ipfs-http-client@d6c8cbd5e643d2131477120a76159074d1a07cd3 --- client/httpapi/api_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5621ab87f..fa38866e1 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -19,9 +19,7 @@ import ( "github.com/ipfs/iptb/testbed/interfaces" ) -type NodeProvider struct{} - -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", NewNode: local.NewNode, @@ -31,8 +29,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) BuiltIn: true, }, false) if err != nil { - return nil, err + panic(err) } +} + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { From 9a6ee6f5e153b96afda0f4ca1ee893cf28aa4f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 17:30:04 +0100 Subject: [PATCH 042/112] Improve test node spawning This commit was moved from ipfs/go-ipfs-http-client@6bb2a287a6f5090792e07f3e3a74871c459b395f --- client/httpapi/api_test.go | 198 ++++++++++++++++++++++++------------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fa38866e1..af2180384 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,23 +2,23 @@ package httpapi import ( "context" - "fmt" "io/ioutil" gohttp "net/http" "os" - "path" "strconv" + "sync" "testing" "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" - "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" "github.com/ipfs/iptb/testbed/interfaces" + ma "github.com/multiformats/go-multiaddr" ) +const parallelSpeculativeNodes = 15 // 15 seems to work best + func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", @@ -33,99 +33,163 @@ func init() { } } -type NodeProvider struct{} +type NodeProvider struct { + simple <-chan func(context.Context) ([]iface.CoreAPI, error) +} -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func newNodeProvider(ctx context.Context) *NodeProvider { + simpleNodes := make(chan func(context.Context) ([]iface.CoreAPI, error), parallelSpeculativeNodes) + + np := &NodeProvider{ + simple: simpleNodes, + } + + // start basic nodes speculatively in parallel + for i := 0; i < parallelSpeculativeNodes; i++ { + go func() { + for { + ctx, cancel := context.WithCancel(ctx) + + snd, err := np.makeAPISwarm(ctx, false, 1) + + res := func(ctx context.Context) ([]iface.CoreAPI, error) { + if err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + cancel() + }() + + return snd, nil + } + + select { + case simpleNodes <- res: + case <-ctx.Done(): + return + } + } + }() + } + + return np +} + +func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + if !fullIdentity && n == 1 { + return (<-np.simple)(ctx) + } + return np.makeAPISwarm(ctx, fullIdentity, n) +} + +func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { return nil, err } - c := cli.NewCli() //TODO: is there a better way? + tb := testbed.NewTestbed(dir) - initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} - if err := c.Run(initArgs); err != nil { + specs, err := testbed.BuildSpecs(tb.Dir(), n, "localipfs", nil) + if err != nil { return nil, err } - filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} - if err := c.Run(filestoreArgs); err != nil { + if err := testbed.WriteNodeSpecs(tb.Dir(), specs); err != nil { return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} - if err := c.Run(startArgs); err != nil { + nodes, err := tb.Nodes() + if err != nil { return nil, err } - if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} - if err := c.Run(connectArgs); err != nil { - return nil, err - } + apis := make([]iface.CoreAPI, n) + + wg := sync.WaitGroup{} + zero := sync.WaitGroup{} + + wg.Add(len(nodes)) + zero.Add(1) + + for i, nd := range nodes { + go func(i int, nd testbedi.Core) { + defer wg.Done() + + if _, err := nd.Init(ctx, "--empty-repo"); err != nil { + panic(err) + } + + if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { + panic(err) + } + + if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { + panic(err) + } + + if i > 0 { + zero.Wait() + if err := nd.Connect(ctx, nodes[0]); err != nil { + panic(err) + } + } else { + zero.Done() + } + + addr, err := nd.APIAddr() + if err != nil { + panic(err) + } + + maddr, err := ma.NewMultiaddr(addr) + if err != nil { + panic(err) + } + + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(maddr, c) + + // empty node is pinned even with --empty-repo, we don't want that + emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + panic(err) + } + if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { + panic(err) + } + }(i, nd) } + wg.Wait() + go func() { <-ctx.Done() defer os.Remove(dir) defer func() { - _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + for _, nd := range nodes { + _ = nd.Stop(context.Background()) + } }() }() - apis := make([]iface.CoreAPI, n) - - for i := range apis { - tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) - - node, err := tb.Node(i) - if err != nil { - return nil, err - } - - attrNode, ok := node.(testbedi.Attribute) - if !ok { - return nil, fmt.Errorf("node does not implement attributes") - } - - pth, err := attrNode.Attr("path") - if err != nil { - return nil, err - } - - a := ApiAddr(pth) - if a == nil { - return nil, fmt.Errorf("nil addr for node") - } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - DisableCompression: true, - }, - } - apis[i] = NewApiWithClient(a, c) - - // node cleanup - // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) - if err != nil { - return nil, err - } - for _, pin := range pins { //TODO: parallel - if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { - return nil, err - } - } - - } - return apis, nil } func TestHttpApi(t *testing.T) { - tests.TestApi(&NodeProvider{})(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests.TestApi(newNodeProvider(ctx))(t) } From f7dd0c690960ea79a7aad7f40b11362512d173bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:12:02 +0100 Subject: [PATCH 043/112] Improve apifile error messages This commit was moved from ipfs/go-ipfs-http-client@3393b8379021695126da83afdf3bf54eaff828f4 --- client/httpapi/apifile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1e5f61a9a..864760684 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -157,12 +157,12 @@ func (it *apiIter) Next() bool { } if len(out.Objects) != 1 { - it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects)) return false } if len(out.Objects[0].Links) != 1 { - it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links)) return false } From e1b14d78c72edf0a782e362afb0e818e032c31cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:45:19 +0100 Subject: [PATCH 044/112] Implement missing methods This commit was moved from ipfs/go-ipfs-http-client@b76413dfe55fd33d39a680458282d85565cc1c69 --- client/httpapi/api.go | 3 -- client/httpapi/name.go | 63 +++++++++++++++++++++++++++++++++++++++--- client/httpapi/path.go | 9 ++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 698e36524..e106d8d87 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,6 @@ package httpapi import ( - "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -23,8 +22,6 @@ const ( EnvDir = "IPFS_PATH" ) -var ErrNotImplemented = errors.New("not implemented") - type HttpApi struct { url string httpcli gohttp.Client diff --git a/client/httpapi/name.go b/client/httpapi/name.go index d760e6188..58fb59249 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "fmt" + "io" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -38,11 +40,11 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na req := api.core().request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). - Option("lifetime", options.ValidTime.String()). + Option("lifetime", options.ValidTime). Option("resolve", false) if options.TTL != nil { - req.Option("ttl", options.TTL.String()) + req.Option("ttl", options.TTL) } var out ipnsEntry @@ -57,7 +59,60 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { - return nil, ErrNotImplemented + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } + + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout). + Option("stream", true) + resp, err := req.Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + res := make(chan iface.IpnsResult) + + go func() { + defer close(res) + defer resp.Close() + + dec := json.NewDecoder(resp.Output) + + for { + var out struct{ Path string } + err := dec.Decode(&out) + if err == io.EOF { + return + } + var ires iface.IpnsResult + if err == nil { + ires.Path, err = iface.ParsePath(out.Path) + } + + select { + case res <- ires: + case <-ctx.Done(): + } + if err != nil { + return + } + } + }() + + return res, nil } func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { @@ -75,7 +130,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). - Option("dht-timeout", ropts.DhtTimeout.String()) + Option("dht-timeout", ropts.DhtTimeout) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 68dc2633d..8c819121a 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -42,6 +42,11 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { - return nil, ErrNotImplemented +func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + return api.Dag().Get(ctx, rp.Cid()) } From 42273cab06348b19a813dee77f7cbe5ed7cc78d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:05:17 +0100 Subject: [PATCH 045/112] Don't use valid() pattern This commit was moved from ipfs/go-ipfs-http-client@4d07c48f98b0af0b18c6c13f80f04c966fc43bdc --- client/httpapi/block.go | 20 +++++++------------- client/httpapi/key.go | 28 ++++++++++++++++------------ client/httpapi/name.go | 14 ++++---------- client/httpapi/pubsub.go | 12 +++++------- 4 files changed, 32 insertions(+), 42 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 45c73472c..fd4d9bab9 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -18,24 +18,16 @@ type BlockAPI HttpApi type blockStat struct { Key string BSize int `json:"Size"` + + cid cid.Cid } func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) valid() (iface.ResolvedPath, error) { - c, err := cid.Parse(s.Key) - if err != nil { - return nil, err - } - - return iface.IpldPath(c), nil -} - func (s *blockStat) Path() iface.ResolvedPath { - p, _ := s.valid() - return p + return iface.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +52,8 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } @@ -118,7 +111,8 @@ func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, e if err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index dc2adf8b7..a16c30d8e 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -14,6 +14,8 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` Id string + + pid peer.ID } func (k *keyOutput) Name() string { @@ -26,13 +28,7 @@ func (k *keyOutput) Path() iface.Path { } func (k *keyOutput) ID() peer.ID { - p, _ := peer.IDB58Decode(k.Id) - return p -} - -func (k *keyOutput) valid() error { - _, err := peer.IDB58Decode(k.Id) - return err + return k.pid } func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { @@ -49,7 +45,8 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { @@ -72,7 +69,8 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - return id, out.Overwrite, id.valid() + id.pid, err = peer.IDB58Decode(id.Id) + return id, out.Overwrite, err } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { @@ -83,7 +81,9 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { - if err := k.valid(); err != nil { + var err error + k.pid, err = peer.IDB58Decode(k.Id) + if err != nil { return nil, err } res[i] = k @@ -98,8 +98,10 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { return nil, err } + var err error out := keyOutput{JName: "self", Id: id.ID} - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { @@ -111,7 +113,9 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { return nil, errors.New("got unexpected number of keys back") } - return &out.Keys[0], out.Keys[0].valid() + var err error + out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + return &out.Keys[0], err } func (api *KeyAPI) core() *HttpApi { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 58fb59249..b848aa819 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -16,10 +16,8 @@ type NameAPI HttpApi type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` -} -func (e *ipnsEntry) valid() (iface.Path, error) { - return iface.ParsePath(e.JValue) + path iface.Path } func (e *ipnsEntry) Name() string { @@ -27,8 +25,7 @@ func (e *ipnsEntry) Name() string { } func (e *ipnsEntry) Value() iface.Path { - p, _ := e.valid() - return p + return e.path } func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { @@ -51,11 +48,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { - return nil, err - } - - return &out, nil + out.path, err = iface.ParsePath(out.JValue) + return &out, err } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 334509780..e27cb5655 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -66,16 +66,12 @@ type pubsubMessage struct { JData []byte `json:"data,omitempty"` JSeqno []byte `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` -} -func (msg *pubsubMessage) valid() error { - _, err := peer.IDFromBytes(msg.JFrom) - return err + from peer.ID } func (msg *pubsubMessage) From() peer.ID { - id, _ := peer.IDFromBytes(msg.JFrom) - return id + return msg.from } func (msg *pubsubMessage) Data() []byte { @@ -97,7 +93,9 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if err := s.dec.Decode(&msg); err != nil { return nil, err } - return &msg, msg.valid() + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { From 93f684617a3adbd7cba751ee421e32b75cd3dafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:07:45 +0100 Subject: [PATCH 046/112] dag: remove unused waitgroup This commit was moved from ipfs/go-ipfs-http-client@934fc60a7c41a14b679fd89fec3b349650a76c81 --- client/httpapi/dag.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 3f54ced34..a613d18de 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -39,13 +39,10 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { out := make(chan *format.NodeOption) - wg := sync.WaitGroup{} - wg.Add(len(cids)) for _, c := range cids { // TODO: Consider limiting concurrency of this somehow go func(c cid.Cid) { - defer wg.Done() n, err := api.Get(ctx, c) select { From cc964b4ab8ea5d059be2c05e64ecdca851dcd8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:15:21 +0100 Subject: [PATCH 047/112] Simplify Object.New, remove ipldnode.go This commit was moved from ipfs/go-ipfs-http-client@0752a6ee63a2ec42aabb6c9acac6ef6c4a94af38 --- client/httpapi/dag.go | 1 - client/httpapi/ipldnode.go | 113 ------------------------------------- client/httpapi/object.go | 34 +++++------ 3 files changed, 18 insertions(+), 130 deletions(-) delete mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index a613d18de..669b5f893 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io/ioutil" - "sync" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go deleted file mode 100644 index 43fa5d50d..000000000 --- a/client/httpapi/ipldnode.go +++ /dev/null @@ -1,113 +0,0 @@ -package httpapi - -import ( - "context" - "errors" - "io/ioutil" - "strconv" - - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" -) - -type ipldNode struct { - ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? - path iface.ResolvedPath - api *HttpApi -} - -func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { - return &ipldNode{ - ctx: ctx, - path: p, - api: api, - } -} - -func (n *ipldNode) RawData() []byte { - r, err := n.api.Block().Get(n.ctx, n.path) - if err != nil { - panic(err) // TODO: eww, should we add errors too / better ideas? - } - - b, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - - return b -} - -func (n *ipldNode) Cid() cid.Cid { - return n.path.Cid() -} - -func (n *ipldNode) String() string { - return n.Cid().String() -} - -func (n *ipldNode) Loggable() map[string]interface{} { - return nil //TODO: we can't really do better here, can we? -} - -// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. -// We can also give people a choice -func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { - p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) - - var out interface{} - n.api.request("dag/get", p).Exec(n.ctx, &out) - - // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack - for len(path) > 0 { - switch o := out.(type) { - case map[string]interface{}: - v, ok := o[path[0]] - if !ok { - // TODO: ipld links - return nil, nil, errors.New("no element under this path") - } - out = v - case []interface{}: - n, err := strconv.ParseUint(path[0], 10, 32) - if err != nil { - return nil, nil, err - } - if len(o) < int(n) { - return nil, nil, errors.New("no element under this path") - } - out = o[n] - } - path = path[1:] - } - - return out, path, nil -} - -func (n *ipldNode) Tree(path string, depth int) []string { - panic("implement me") -} - -func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - panic("implement me") -} - -func (n *ipldNode) Copy() ipld.Node { - panic("implement me") -} - -func (n *ipldNode) Links() []*ipld.Link { - panic("implement me") -} - -func (n *ipldNode) Stat() (*ipld.NodeStat, error) { - panic("implement me") -} - -func (n *ipldNode) Size() (uint64, error) { - panic("implement me") -} - -var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 3b648d82b..5a06f74d9 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -3,12 +3,15 @@ package httpapi import ( "bytes" "context" + "fmt" "io" "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + dag "github.com/ipfs/go-merkledag" + ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" ) @@ -19,24 +22,23 @@ type objectOut struct { Hash string } -func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) { options, err := caopts.ObjectNewOptions(opts...) if err != nil { return nil, err } - var out objectOut - err = api.core().request("object/new", options.Type).Exec(ctx, &out) - if err != nil { - return nil, err + var n ipld.Node + switch options.Type { + case "empty": + n = new(dag.ProtoNode) + case "unixfs-dir": + n = ft.EmptyDirNode() + default: + return nil, fmt.Errorf("unknown object type: %s", options.Type) } - c, err := cid.Parse(out.Hash) - if err != nil { - return nil, err - } - - return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil + return n, nil } func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { @@ -64,7 +66,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -96,7 +98,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -107,14 +109,14 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } - res := make([]*format.Link, len(out.Links)) + res := make([]*ipld.Link, len(out.Links)) for i, l := range out.Links { c, err := cid.Parse(l.Hash) if err != nil { return nil, err } - res[i] = &format.Link{ + res[i] = &ipld.Link{ Cid: c, Name: l.Name, Size: l.Size, From a23da822de54922faf1f82a86c25660e8cca8bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:35:32 +0100 Subject: [PATCH 048/112] swarm: attach peerid if needed This commit was moved from ipfs/go-ipfs-http-client@aa88b18ac497400d16b36af6489af50a003c298c --- client/httpapi/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index d179b6540..0bb36aca3 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -17,6 +17,13 @@ type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { + if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + addr = addr.Encapsulate(pidma) + } saddrs[i] = addr.String() } From f005b8dc5666916ff2a0ff1eef58b7795670ec73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 16:39:00 +0100 Subject: [PATCH 049/112] pin: verify: parse bad node cids early This commit was moved from ipfs/go-ipfs-http-client@7032dfc9b016f2c584e2102e80fb60c3ecf573ea --- client/httpapi/pin.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 4c4e5713c..b11efd35c 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -110,23 +110,18 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { type badNode struct { Cid string JErr string `json:"Err"` + + cid cid.Cid } func (n *badNode) Path() iface.ResolvedPath { - c, err := cid.Parse(n.Cid) - if err != nil { - return nil // todo: handle this better - } - return iface.IpldPath(c) + return iface.IpldPath(n.cid) } func (n *badNode) Err() error { if n.JErr != "" { return errors.New(n.JErr) } - if _, err := cid.Parse(n.Cid); err != nil { - return err - } return nil } @@ -150,6 +145,13 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { return // todo: handle non io.EOF somehow } + for i, n := range out.JBadNodes { + out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + if err != nil { + return + } + } + select { case res <- &out: case <-ctx.Done(): From 4a6d36d98b6527e9c13918ac901c4cbc68630b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:02:13 +0100 Subject: [PATCH 050/112] pubsub: handle ctx This commit was moved from ipfs/go-ipfs-http-client@d451a4943c5311d7e32fcf86f56bdc1eda977fc7 --- client/httpapi/pubsub.go | 70 ++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index e27cb5655..edc1a9709 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -57,8 +57,10 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) } type pubsubSub struct { - io.Closer - dec *json.Decoder + messages chan pubsubMessage + + done chan struct{} + rcloser io.Closer } type pubsubMessage struct { @@ -68,6 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID + err error } func (msg *pubsubMessage) From() peer.ID { @@ -87,15 +90,20 @@ func (msg *pubsubMessage) Topics() []string { } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { - // TODO: handle ctx - - var msg pubsubMessage - if err := s.dec.Decode(&msg); err != nil { - return nil, err + select { + case msg, ok := <-s.messages: + if !ok { + return nil, io.EOF + } + if msg.err != nil { + return nil, msg.err + } + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err + case <-ctx.Done(): + return nil, ctx.Err() } - var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { @@ -114,10 +122,44 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, resp.Error } - return &pubsubSub{ - Closer: resp, - dec: json.NewDecoder(resp.Output), - }, nil + sub := &pubsubSub{ + messages: make(chan pubsubMessage), + done: make(chan struct{}), + } + + dec := json.NewDecoder(resp.Output) + + go func() { + defer close(sub.messages) + + for { + var msg pubsubMessage + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + return + } + msg.err = err + } + + select { + case sub.messages <- msg: + case <-sub.done: + return + case <-ctx.Done(): + return + } + } + }() + + return sub, nil +} + +func (s*pubsubSub) Close() error { + if s.done != nil { + close(s.done) + s.done = nil + } + return s.rcloser.Close() } func (api *PubsubAPI) core() *HttpApi { From 7229dbbf73fd925da390fc5db5a25caf398b1fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:04:33 +0100 Subject: [PATCH 051/112] don't read all and then throw away the buffer This commit was moved from ipfs/go-ipfs-http-client@5c96c2954a5d45dd907dca9942d2773d76b8a71b --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 745c9a2a9..b9e83eb3d 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -139,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - ioutil.ReadAll(resp.Body) + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } From b6ace8dd401321b842c5f0054d9ed025fd19cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:12:37 +0100 Subject: [PATCH 052/112] response: handle late errors This commit was moved from ipfs/go-ipfs-http-client@139e9e5ff1b52a92c169d28bf56ff58e36b49ca3 --- client/httpapi/requestbuilder.go | 4 ++-- client/httpapi/response.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 628ad03cd..af43ce236 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -103,11 +103,11 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { } if res == nil { - httpRes.Close() + lateErr := httpRes.Close() if httpRes.Error != nil { return httpRes.Error } - return nil + return lateErr } return httpRes.Decode(res) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index b9e83eb3d..f773130e8 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -39,9 +39,14 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data - return r.Output.Close() + + // always drain output (response body) //TODO: make optional for things like cat + _, err1 := io.Copy(ioutil.Discard, r.Output) + err2 := r.Output.Close() + if err1 != nil { + return err1 + } + return err2 } return nil } From 7be8d01ee732be71f08f4482e0d5c5b61731ee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:27:12 +0100 Subject: [PATCH 053/112] response: option to disable output draining This commit was moved from ipfs/go-ipfs-http-client@9f3d9635fa4e977b6ec202c9e4f78ca1c64fc0dd --- client/httpapi/api.go | 1 + client/httpapi/apifile.go | 2 +- client/httpapi/pubsub.go | 4 ++-- client/httpapi/request.go | 2 ++ client/httpapi/requestbuilder.go | 7 +++++++ client/httpapi/response.go | 10 ++++++++-- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e106d8d87..f74300216 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -131,6 +131,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, + drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 864760684..e3cb85ea4 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,7 +57,7 @@ func (f *apiFile) reset() error { if f.r != nil { f.r.Close() } - req := f.core.request("cat", f.path.String()) + req := f.core.request("cat", f.path.String()).NoDrain() if f.at != 0 { req.Option("offset", f.at) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index edc1a9709..b3e45ed36 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -113,8 +113,8 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover). - Send(ctx) + Option("discover", options.Discover).NoDrain().Send(ctx) + if err != nil { return nil, err } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67..18cfb7fd0 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,6 +13,7 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string + DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -30,5 +31,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), + DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236..8b040522e 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,6 +19,7 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader + drainOut bool shell *HttpApi } @@ -84,6 +85,12 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } +// NoDrain disables output draining in response closer +func (r *RequestBuilder) NoDrain() *RequestBuilder { + r.drainOut = false + return r +} + // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f773130e8..cd3cc2b71 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -35,13 +35,18 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error + + drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) //TODO: make optional for things like cat - _, err1 := io.Copy(ioutil.Discard, r.Output) + // always drain output (response body) + var err1 error + if r.drainOutput { + _, err1 = io.Copy(ioutil.Discard, r.Output) + } err2 := r.Output.Close() if err1 != nil { return err1 @@ -114,6 +119,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) + nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ From 3088776f8150fe6967a245cf0360f5219e2dc9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:30:02 +0100 Subject: [PATCH 054/112] swarm: always append peer IDs This commit was moved from ipfs/go-ipfs-http-client@49267901c71f77f098deaf8966aa2dcecf22c2df --- client/httpapi/swarm.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0bb36aca3..13a814bc4 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -15,16 +15,14 @@ import ( type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { - if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { - pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) - if err != nil { - return err - } - addr = addr.Encapsulate(pidma) - } - saddrs[i] = addr.String() + saddrs[i] = addr.Encapsulate(pidma).String() } return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) From 745bf92506bd488bde4f695c4d644e7d3b58f0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:32:40 +0100 Subject: [PATCH 055/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@f3c2c350861470c05617cf74e7b78d06782bb706 --- client/httpapi/api.go | 6 +++--- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f74300216..6dbfe3b6a 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -128,9 +128,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, drainOut: true, } } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index b3e45ed36..49c58ab88 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -59,7 +59,7 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) type pubsubSub struct { messages chan pubsubMessage - done chan struct{} + done chan struct{} rcloser io.Closer } @@ -70,7 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID - err error + err error } func (msg *pubsubMessage) From() peer.ID { @@ -124,7 +124,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), - done: make(chan struct{}), + done: make(chan struct{}), } dec := json.NewDecoder(resp.Output) @@ -154,7 +154,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return sub, nil } -func (s*pubsubSub) Close() error { +func (s *pubsubSub) Close() error { if s.done != nil { close(s.done) s.done = nil diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 18cfb7fd0..d65d63835 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string DrainOut bool // if set, resp.Close will read all remaining data } @@ -26,11 +26,11 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 8b040522e..c5bc44124 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader drainOut bool shell *HttpApi From 27aa13fe44561e8d9469ce537e64baa60d17a705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 21:53:17 +0100 Subject: [PATCH 056/112] return errors from constructor methods This commit was moved from ipfs/go-ipfs-http-client@dbee4e27aaf9ef8e3fa7e25ac1f4f28df8a31e0d --- client/httpapi/api.go | 39 ++++++++++++++++---------------------- client/httpapi/api_test.go | 5 ++++- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6dbfe3b6a..e4603bd38 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,8 +29,7 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } -//TODO: Return errors here -func NewLocalApi() iface.CoreAPI { +func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -39,40 +38,34 @@ func NewLocalApi() iface.CoreAPI { return NewPathApi(baseDir) } -func NewPathApi(p string) iface.CoreAPI { - a := ApiAddr(p) - if a == nil { - return nil +func NewPathApi(p string) (iface.CoreAPI, error) { + a, err := ApiAddr(p) + if err != nil { + if err == os.ErrNotExist { + err = nil + } + return nil, err } return NewApi(a) } -func ApiAddr(ipfspath string) ma.Multiaddr { +func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { - return nil + return nil, err } apiFile := path.Join(baseDir, DefaultApiFile) - if _, err := os.Stat(apiFile); err != nil { - return nil - } - api, err := ioutil.ReadFile(apiFile) if err != nil { - return nil + return nil, err } - maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) - if err != nil { - return nil - } - - return maddr + return ma.NewMultiaddr(strings.TrimSpace(string(api))) } -func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? +func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -83,10 +76,10 @@ func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? return NewApiWithClient(a, c) } -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { - return nil // TODO: return that error + return nil, err } if a, err := ma.NewMultiaddr(url); err == nil { @@ -107,7 +100,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return fmt.Errorf("unexpected redirect") } - return api + return api, nil } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index af2180384..2f6d47c58 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -157,7 +157,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) DisableCompression: true, }, } - apis[i] = NewApiWithClient(maddr, c) + apis[i], err = NewApiWithClient(maddr, c) + if err != nil { + panic(err) + } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") From fc299e7e849d46e364204234f9ea3d3aa52e42cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:19:56 +0100 Subject: [PATCH 057/112] pin verify: use temporary struct This commit was moved from ipfs/go-ipfs-http-client@e400fa3b380b589c2ae385a66a007d685147e4c7 --- client/httpapi/pin.go | 57 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index b11efd35c..f667a5487 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -90,27 +90,20 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` - JBadNodes []*badNode `json:"BadNodes,omitempty"` + ok bool + badNodes []iface.BadPinNode } func (r *pinVerifyRes) Ok() bool { - return r.JOk + return r.ok } func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { - out := make([]iface.BadPinNode, len(r.JBadNodes)) - for i, n := range r.JBadNodes { - out[i] = n - } - return out + return r.badNodes } type badNode struct { - Cid string - JErr string `json:"Err"` - + err error cid cid.Cid } @@ -119,10 +112,7 @@ func (n *badNode) Path() iface.ResolvedPath { } func (n *badNode) Err() error { - if n.JErr != "" { - return errors.New(n.JErr) - } - return nil + return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { @@ -140,20 +130,45 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { defer close(res) dec := json.NewDecoder(resp.Output) for { - var out pinVerifyRes + var out struct { + Cid string + Ok bool + + BadNodes []struct{ + Cid string + Err string + } + } if err := dec.Decode(&out); err != nil { return // todo: handle non io.EOF somehow } - for i, n := range out.JBadNodes { - out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + badNodes := make([]iface.BadPinNode, len(out.BadNodes)) + for i, n := range out.BadNodes { + c, err := cid.Decode(n.Cid) if err != nil { - return + badNodes[i] = &badNode{ + cid: c, + err: err, + } + continue + } + + if n.Err != "" { + err = errors.New(n.Err) + } + badNodes[i] = &badNode{ + cid: c, + err: err, } } select { - case res <- &out: + case res <- &pinVerifyRes{ + ok: out.Ok, + + badNodes: badNodes, + }: case <-ctx.Done(): return } From a058e7d31a32d1bca98e41886a310149b872af84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:27:48 +0100 Subject: [PATCH 058/112] response: Document zero-result Decode behaviour This commit was moved from ipfs/go-ipfs-http-client@902bc5ef8c1b9595e97728dca3da0d0da124f142 --- client/httpapi/response.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index cd3cc2b71..26f549c9a 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -71,6 +71,9 @@ func (r *Response) Decode(dec interface{}) error { } n++ } + + // Decode expects at least one result. For calls where zero results are valid, + // use Send and construct json Decoder manually. if n > 0 && err == io.EOF { err = nil } From db1af499c9d661a70ac6fcce9cc7347b934385e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:33:57 +0100 Subject: [PATCH 059/112] use mime.ParseMediaType for Content-Type response parsing This commit was moved from ipfs/go-ipfs-http-client@217d1945237321c12f96abef82707217a30813b0 --- client/httpapi/response.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 26f549c9a..708d2714f 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,14 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" + "mime" "net/http" "net/url" "os" - "strings" - - files "github.com/ipfs/go-ipfs-files" ) type trailerReader struct { @@ -116,9 +115,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType := resp.Header.Get("Content-Type") - parts := strings.Split(contentType, ";") - contentType = parts[0] + + contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } nresp := new(Response) From f34788e6ee61aad0ee1970a4f3d004bdebdbcd27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:11:08 +0100 Subject: [PATCH 060/112] cleanup Swarm.Peers This commit was moved from ipfs/go-ipfs-http-client@fd7858dc57b86761599b0e844f3536f2d4a0d953 --- client/httpapi/api.go | 1 - client/httpapi/apifile.go | 14 +++--- client/httpapi/pubsub.go | 9 ++-- client/httpapi/request.go | 2 - client/httpapi/requestbuilder.go | 7 --- client/httpapi/response.go | 19 +++++--- client/httpapi/swarm.go | 83 +++++++++++++++++--------------- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e4603bd38..c5a706d05 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -124,7 +124,6 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, - drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e3cb85ea4..a8eb0de1a 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -49,15 +49,15 @@ type apiFile struct { size int64 path iface.Path - r io.ReadCloser + r *Response at int64 } func (f *apiFile) reset() error { if f.r != nil { - f.r.Close() + f.r.Cancel() } - req := f.core.request("cat", f.path.String()).NoDrain() + req := f.core.request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -68,12 +68,12 @@ func (f *apiFile) reset() error { if resp.Error != nil { return resp.Error } - f.r = resp.Output + f.r = resp return nil } func (f *apiFile) Read(p []byte) (int, error) { - n, err := f.r.Read(p) + n, err := f.r.Output.Read(p) if n > 0 { f.at += int64(n) } @@ -92,7 +92,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) + r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err @@ -103,7 +103,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { func (f *apiFile) Close() error { if f.r != nil { - return f.r.Close() + return f.r.Cancel() } return nil } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 49c58ab88..2ac04b53c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -60,7 +60,7 @@ type pubsubSub struct { messages chan pubsubMessage done chan struct{} - rcloser io.Closer + rcloser func() error } type pubsubMessage struct { @@ -113,7 +113,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover).NoDrain().Send(ctx) + Option("discover", options.Discover).Send(ctx) if err != nil { return nil, err @@ -125,6 +125,9 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), done: make(chan struct{}), + rcloser: func() error { + return resp.Cancel() + }, } dec := json.NewDecoder(resp.Output) @@ -159,7 +162,7 @@ func (s *pubsubSub) Close() error { close(s.done) s.done = nil } - return s.rcloser.Close() + return s.rcloser() } func (api *PubsubAPI) core() *HttpApi { diff --git a/client/httpapi/request.go b/client/httpapi/request.go index d65d63835..f6f7ee486 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,7 +13,6 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string - DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -31,6 +30,5 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), - DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index c5bc44124..ba407217f 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,7 +19,6 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader - drainOut bool shell *HttpApi } @@ -85,12 +84,6 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } -// NoDrain disables output draining in response closer -func (r *RequestBuilder) NoDrain() *RequestBuilder { - r.drainOut = false - return r -} - // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 708d2714f..c9f178f9c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -34,18 +34,13 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error - - drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - var err1 error - if r.drainOutput { - _, err1 = io.Copy(ioutil.Discard, r.Output) - } + // drain output (response body) + _, err1 := io.Copy(ioutil.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -55,6 +50,15 @@ func (r *Response) Close() error { return nil } +// Cancel aborts running request (without draining request body) +func (r *Response) Cancel() error { + if r.Output != nil { + return r.Output.Close() + } + + return nil +} + func (r *Response) Decode(dec interface{}) error { defer r.Close() if r.Error != nil { @@ -123,7 +127,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 13a814bc4..882711917 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -32,74 +32,81 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) } -type streamInfo struct { - Protocol string -} - type connInfo struct { - Addr string - Peer string - JLatency time.Duration `json:"Latency"` - Muxer string - JDirection inet.Direction `json:"Direction"` - JStreams []streamInfo `json:"Streams"` -} - -func (c *connInfo) valid() error { - _, err := multiaddr.NewMultiaddr(c.Addr) - if err != nil { - return err - } - - _, err = peer.IDB58Decode(c.Peer) - return err + addr multiaddr.Multiaddr + peer peer.ID + latency time.Duration + muxer string + direction inet.Direction + streams []protocol.ID } func (c *connInfo) ID() peer.ID { - id, _ := peer.IDB58Decode(c.Peer) - return id + return c.peer } func (c *connInfo) Address() multiaddr.Multiaddr { - a, _ := multiaddr.NewMultiaddr(c.Addr) - return a + return c.addr } func (c *connInfo) Direction() inet.Direction { - return c.JDirection + return c.direction } func (c *connInfo) Latency() (time.Duration, error) { - return c.JLatency, nil + return c.latency, nil } func (c *connInfo) Streams() ([]protocol.ID, error) { - res := make([]protocol.ID, len(c.JStreams)) - for i, stream := range c.JStreams { - res[i] = protocol.ID(stream.Protocol) - } - return res, nil + return c.streams, nil } func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { - var out struct { - Peers []*connInfo + var resp struct { + Peers []struct{ + Addr string + Peer string + Latency time.Duration + Muxer string + Direction inet.Direction + Streams []struct { + Protocol string + } + } } err := api.core().request("swarm/peers"). Option("streams", true). Option("latency", true). - Exec(ctx, &out) + Exec(ctx, &resp) if err != nil { return nil, err } - res := make([]iface.ConnectionInfo, len(out.Peers)) - for i, conn := range out.Peers { - if err := conn.valid(); err != nil { + res := make([]iface.ConnectionInfo, len(resp.Peers)) + for i, conn := range resp.Peers { + out := &connInfo{ + latency: conn.Latency, + muxer: conn.Muxer, + direction: conn.Direction, + } + + out.peer, err = peer.IDB58Decode(conn.Peer) + if err != nil { return nil, err } - res[i] = conn + + out.addr, err = multiaddr.NewMultiaddr(conn.Addr) + if err != nil { + return nil, err + } + + out.streams = make([]protocol.ID, len(conn.Streams)) + for i, p := range conn.Streams { + out.streams[i] = protocol.ID(p.Protocol) + } + + res[i] = out } return res, nil From aa4d6e16f13bd753b77d0a8fc2174e41af89310e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:26:34 +0100 Subject: [PATCH 061/112] Add some docs to constructors This commit was moved from ipfs/go-ipfs-http-client@e34cd600e690bfa9bd20d6089b9335ac6090d4dd --- client/httpapi/api.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c5a706d05..9e8d55188 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -22,6 +22,11 @@ const ( EnvDir = "IPFS_PATH" ) +// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using +// IPFS HTTP API. +// +// For interface docs see +// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string httpcli gohttp.Client @@ -29,6 +34,11 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } +// NewLocalApi tries to construct new HttpApi instance communicating with local +// IPFS daemon +// +// Daemon api address is pulled from the $IPFS_PATH/api file. +// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -38,8 +48,10 @@ func NewLocalApi() (iface.CoreAPI, error) { return NewPathApi(baseDir) } -func NewPathApi(p string) (iface.CoreAPI, error) { - a, err := ApiAddr(p) +// NewPathApi constructs new HttpApi by pulling api address from specified +// ipfspath. Api file should be located at $ipfspath/api +func NewPathApi(ipfspath string) (iface.CoreAPI, error) { + a, err := ApiAddr(ipfspath) if err != nil { if err == os.ErrNotExist { err = nil @@ -49,6 +61,7 @@ func NewPathApi(p string) (iface.CoreAPI, error) { return NewApi(a) } +// ApiAddr reads api file in specified ipfs path func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { @@ -65,6 +78,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return ma.NewMultiaddr(strings.TrimSpace(string(api))) } +// NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ @@ -76,6 +90,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { return NewApiWithClient(a, c) } +// NewApiWithClient constructs HttpApi with specified endpoint and custom http client func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { From f6a3a4fb2d715ffc0394408287b23eae0d322631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:28:39 +0100 Subject: [PATCH 062/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@65cd935e13a2ea05befbc69d72d64a89056952ea --- client/httpapi/api.go | 6 +++--- client/httpapi/pin.go | 4 ++-- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- client/httpapi/response.go | 1 - client/httpapi/swarm.go | 18 +++++++++--------- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 9e8d55188..b67fb0b48 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -136,9 +136,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, } } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f667a5487..0111d626a 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -132,9 +132,9 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { for { var out struct { Cid string - Ok bool + Ok bool - BadNodes []struct{ + BadNodes []struct { Cid string Err string } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index f6f7ee486..58c61ac67 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -25,10 +25,10 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index ba407217f..af43ce236 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader shell *HttpApi } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index c9f178f9c..84f28214d 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -119,7 +119,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return nil, err diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 882711917..0814debee 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -33,10 +33,10 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e } type connInfo struct { - addr multiaddr.Multiaddr - peer peer.ID + addr multiaddr.Multiaddr + peer peer.ID latency time.Duration - muxer string + muxer string direction inet.Direction streams []protocol.ID } @@ -63,11 +63,11 @@ func (c *connInfo) Streams() ([]protocol.ID, error) { func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { var resp struct { - Peers []struct{ - Addr string - Peer string + Peers []struct { + Addr string + Peer string Latency time.Duration - Muxer string + Muxer string Direction inet.Direction Streams []struct { Protocol string @@ -86,8 +86,8 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { out := &connInfo{ - latency: conn.Latency, - muxer: conn.Muxer, + latency: conn.Latency, + muxer: conn.Muxer, direction: conn.Direction, } From ad844e3d0b0221333c04896ed80f11afed301495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 16:50:38 +0100 Subject: [PATCH 063/112] response: simplify Decode This commit was moved from ipfs/go-ipfs-http-client@adbfda4c1c6ee037df3f91ca73f60de1f4620d67 --- client/httpapi/api.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 20 +++----------------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b67fb0b48..17a289cc5 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -53,7 +53,7 @@ func NewLocalApi() (iface.CoreAPI, error) { func NewPathApi(ipfspath string) (iface.CoreAPI, error) { a, err := ApiAddr(ipfspath) if err != nil { - if err == os.ErrNotExist { + if os.IsNotExist(err) { err = nil } return nil, err diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236..2ffed7a0a 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -110,5 +110,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return lateErr } - return httpRes.Decode(res) + return httpRes.decode(res) } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 84f28214d..794cd1470 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -59,28 +59,14 @@ func (r *Response) Cancel() error { return nil } -func (r *Response) Decode(dec interface{}) error { +// Decode reads request body and decodes it as json +func (r *Response) decode(dec interface{}) error { defer r.Close() if r.Error != nil { return r.Error } - n := 0 - var err error - for { - err = json.NewDecoder(r.Output).Decode(dec) - if err != nil { - break - } - n++ - } - - // Decode expects at least one result. For calls where zero results are valid, - // use Send and construct json Decoder manually. - if n > 0 && err == io.EOF { - err = nil - } - return err + return json.NewDecoder(r.Output).Decode(dec) } type Error struct { From 19d91fbb259220e91c8887ee8a09218865418720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 22:08:02 +0100 Subject: [PATCH 064/112] response: pass close error in decode This commit was moved from ipfs/go-ipfs-http-client@4a87232ecaa96246f038d15eb7ee6094216a201e --- client/httpapi/response.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 794cd1470..624a488d7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -61,12 +61,17 @@ func (r *Response) Cancel() error { // Decode reads request body and decodes it as json func (r *Response) decode(dec interface{}) error { - defer r.Close() if r.Error != nil { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + err := json.NewDecoder(r.Output).Decode(dec) + err2 := r.Close() + if err != nil { + return err + } + + return err2 } type Error struct { From cc9968d5dda71c21b3c90eb376458cacadb59e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:39:12 +0100 Subject: [PATCH 065/112] request: fix Content-Disposition header in Send This commit was moved from ipfs/go-ipfs-http-client@b7db17c63b69dc4fbb726dc720e9c594fa9ed495 --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 624a488d7..339c73658 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -102,7 +102,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { if fr, ok := r.Body.(*files.MultiFileReader); ok { req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) - req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + req.Header.Set("Content-Disposition", "form-data; name=\"files\"") } resp, err := c.Do(req) From 47b820150bddb9868274ed2c6faf79c6b8cbe369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:46:18 +0100 Subject: [PATCH 066/112] test: don't panic on errors in async node construction This commit was moved from ipfs/go-ipfs-http-client@95ce0f3949da47b2db92508b8c7b0a00f502682b --- client/httpapi/api_test.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2f6d47c58..df45c15af 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -114,27 +114,32 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) wg.Add(len(nodes)) zero.Add(1) + errs := make(chan error, len(nodes)) for i, nd := range nodes { go func(i int, nd testbedi.Core) { defer wg.Done() if _, err := nd.Init(ctx, "--empty-repo"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { - panic(err) + errs <- err + return } if i > 0 { zero.Wait() if err := nd.Connect(ctx, nodes[0]); err != nil { - panic(err) + errs <- err + return } } else { zero.Done() @@ -142,12 +147,14 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) addr, err := nd.APIAddr() if err != nil { - panic(err) + errs <- err + return } maddr, err := ma.NewMultiaddr(addr) if err != nil { - panic(err) + errs <- err + return } c := &gohttp.Client{ @@ -159,16 +166,19 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } apis[i], err = NewApiWithClient(maddr, c) if err != nil { - panic(err) + errs <- err + return } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { - panic(err) + errs <- err + return } if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { - panic(err) + errs <- err + return } }(i, nd) } @@ -187,7 +197,12 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) }() }() - return apis, nil + select { + case err = <-errs: + default: + } + + return apis, err } func TestHttpApi(t *testing.T) { From 0e800c3d8431fdcdc22ee2a2ac3337cb009553c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 18:15:37 +0000 Subject: [PATCH 067/112] update interface-go-ipfs-core and handle breaking changes * No more Hidden, StdinName, and Wrap options. * LsLink -> DirEntry with file types that don't expose internals. This commit was moved from ipfs/go-ipfs-http-client@5a7161eeab7127da4348dda6a6dc16f07b051f36 --- client/httpapi/apifile.go | 13 ++++----- client/httpapi/unixfs.go | 58 ++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index a8eb0de1a..e0c0c9e1b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -8,8 +8,9 @@ import ( "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + iface "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -174,17 +175,13 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case iface.THAMTShard: - fallthrough - case iface.TMetadata: - fallthrough - case iface.TDirectory: + case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case iface.TFile: + case unixfs.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 1f340b657..4f717b129 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,9 +8,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + unixfs_pb "github.com/ipfs/go-unixfs/pb" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -40,15 +41,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). Option("fscache", options.FsCache). - Option("hidden", options.Hidden). Option("inline", options.Inline). Option("inline-limit", options.InlineLimit). Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). Option("silent", options.Silent). - Option("stdin-name", options.StdinName). - Option("wrap-with-directory", options.Wrap). Option("progress", options.Progress) if options.RawLeavesSet { @@ -62,13 +60,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix req.Option("trickle", true) } - switch c := f.(type) { - case files.Directory: - req.Body(files.NewMultiFileReader(c, false)) - case files.File: - d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side - req.Body(files.NewMultiFileReader(d, false)) - } + d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) var out addEvent resp, err := req.Send(ctx) @@ -127,7 +120,8 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type iface.FileType + Type unixfs_pb.Data_DataType + Target string } type lsObject struct { @@ -139,7 +133,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err @@ -158,7 +152,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf } dec := json.NewDecoder(resp.Output) - out := make(chan iface.LsLink) + out := make(chan iface.DirEntry) go func() { defer resp.Close() @@ -171,7 +165,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return } select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return @@ -179,7 +173,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: case <-ctx.Done(): } return @@ -187,7 +181,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects[0].Links) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: case <-ctx.Done(): } return @@ -198,21 +192,29 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf c, err := cid.Decode(l0.Hash) if err != nil { select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink + } + select { - case out <- iface.LsLink{ - Link: &format.Link{ - Cid: c, - Name: l0.Name, - Size: l0.Size, - }, - Size: l0.Size, - Type: l0.Type, + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, }: case <-ctx.Done(): } From e76ecd30384799dfe388671ebd153e88903d9388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 16:54:05 +0000 Subject: [PATCH 068/112] nit: return concrete types This is generally considered the "better" way to do things in go. It also allows us to expose functions like `request` at some point without changing the API interface itself. This commit was moved from ipfs/go-ipfs-http-client@524e5d82e6bcd2c4956151a4811addad656d7bf0 --- client/httpapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 17a289cc5..ef6eff3e8 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -39,7 +39,7 @@ type HttpApi struct { // // Daemon api address is pulled from the $IPFS_PATH/api file. // If $IPFS_PATH env var is not present, it defaults to ~/.ipfs -func NewLocalApi() (iface.CoreAPI, error) { +func NewLocalApi() (*HttpApi, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -50,7 +50,7 @@ func NewLocalApi() (iface.CoreAPI, error) { // NewPathApi constructs new HttpApi by pulling api address from specified // ipfspath. Api file should be located at $ipfspath/api -func NewPathApi(ipfspath string) (iface.CoreAPI, error) { +func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { From c6f704b453cd72734ca03793cd370021f4a39db9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:02:04 +0000 Subject: [PATCH 069/112] return an error when the API file isn't found fixes #7 This commit was moved from ipfs/go-ipfs-http-client@897d1b1bcdbf9f6616991366e66ac55e18a99a9c --- client/httpapi/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ef6eff3e8..6c41ef3ab 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,6 +1,7 @@ package httpapi import ( + "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -22,6 +23,9 @@ const ( EnvDir = "IPFS_PATH" ) +// ErrApiNotFound if we fail to find a running daemon. +var ErrApiNotFound = errors.New("ipfs api address could not be found") + // HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using // IPFS HTTP API. // @@ -54,7 +58,7 @@ func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { - err = nil + err = ErrApiNotFound } return nil, err } From 40f40b5a6593af189b81f43e393316b381bd541d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:04:33 +0000 Subject: [PATCH 070/112] fix golang-ci lint nits This commit was moved from ipfs/go-ipfs-http-client@ccf20b0ef5f440aca754f94325fca7202275ba8b --- client/httpapi/apifile.go | 2 +- client/httpapi/response.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e0c0c9e1b..1fd751ede 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -56,7 +56,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { - f.r.Cancel() + _ = f.r.Cancel() } req := f.core.request("cat", f.path.String()) if f.at != 0 { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658..ed3033886 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -147,8 +147,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + _ = resp.Body.Close() } return nresp, nil From c791d73355e33139e2face5fc96884c5444f96c7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:05:02 +0000 Subject: [PATCH 071/112] avoid canceling multiple times This commit was moved from ipfs/go-ipfs-http-client@fe16408225d4c9b688528f67ffabe08198b53ab5 --- client/httpapi/apifile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1fd751ede..9e6df95c2 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,6 +57,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { _ = f.r.Cancel() + f.r = nil } req := f.core.request("cat", f.path.String()) if f.at != 0 { From 1c07bed90465210c27dba3cda850913cf397b5c5 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Mon, 1 Apr 2019 17:13:39 -0700 Subject: [PATCH 072/112] Export (*HttpApi).request to enable building custom requests This commit was moved from ipfs/go-ipfs-http-client@8c9ed7dbc284b4d112f6805914a43e63f234df78 --- client/httpapi/api.go | 2 +- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 8 ++++---- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 10 +++++----- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 18 +++++++++--------- client/httpapi/path.go | 2 +- client/httpapi/pin.go | 10 +++++----- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/swarm.go | 12 ++++++------ client/httpapi/unixfs.go | 4 ++-- 12 files changed, 46 insertions(+), 46 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6c41ef3ab..dad205fd0 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, args: args, diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 9e6df95c2..6e3433a51 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -29,7 +29,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) + err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -59,7 +59,7 @@ func (f *apiFile) reset() error { _ = f.r.Cancel() f.r = nil } - req := f.core.request("cat", f.path.String()) + req := f.core.Request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -225,7 +225,7 @@ func (d *apiDir) Entries() files.DirIterator { } func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index fd4d9bab9..3389f4c31 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -41,7 +41,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("block/put"). + req := api.core().Request("block/put"). Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). @@ -61,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(ctx) + resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR Error string `json:",omitempty"` }{} - req := api.core().request("block/rm"). + req := api.core().Request("block/rm"). Option("force", options.Force). Arguments(p.String()) @@ -107,7 +107,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { var out blockStat - err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { return nil, err } diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index dc7dd6bea..f7417cb7f 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -18,7 +18,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, Type notif.QueryEventType Responses []peerstore.PeerInfo } - resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { return peerstore.PeerInfo{}, err } @@ -48,7 +48,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return nil, err } - resp, err := api.core().request("dht/findprovs", rp.Cid().String()). + resp, err := api.core().Request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -104,7 +104,7 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", rp.Cid().String()). + return api.core().Request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index a16c30d8e..1571a3350 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -38,7 +38,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key } var out keyOutput - err = api.core().request("key/gen", name). + err = api.core().Request("key/gen", name). Option("type", options.Algorithm). Option("size", options.Size). Exec(ctx, &out) @@ -61,7 +61,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o Id string Overwrite bool } - err = api.core().request("key/rename", oldName, newName). + err = api.core().Request("key/rename", oldName, newName). Option("force", options.Force). Exec(ctx, &out) if err != nil { @@ -75,7 +75,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { var out struct{ Keys []*keyOutput } - if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + if err := api.core().Request("key/list").Exec(ctx, &out); err != nil { return nil, err } @@ -94,7 +94,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var id struct{ ID string } - if err := api.core().request("id").Exec(ctx, &id); err != nil { + if err := api.core().Request("id").Exec(ctx, &id); err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { var out struct{ Keys []keyOutput } - if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil { return nil, err } if len(out.Keys) != 1 { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b848aa819..0641d1a22 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -34,7 +34,7 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na return nil, err } - req := api.core().request("name/publish", p.String()). + req := api.core().Request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). Option("lifetime", options.ValidTime). @@ -63,7 +63,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). @@ -120,7 +120,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 5a06f74d9..a1ee486b3 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -48,7 +48,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec } var out objectOut - err = api.core().request("object/put"). + err = api.core().Request("object/put"). Option("inputenc", options.InputEnc). Option("datafieldenc", options.DataType). Option("pin", options.Pin). @@ -80,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) } func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("object/data", p.String()).Send(ctx) + resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er Size uint64 } } - if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]*ipld.Link, len(out.Links)) @@ -135,7 +135,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat DataSize int CumulativeSize int } - if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/stat", p.String()).Exec(ctx, &out); err != nil { return nil, err } @@ -161,7 +161,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, } var out objectOut - err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + err = api.core().Request("object/patch/add-link", base.String(), name, child.String()). Option("create", options.Create). Exec(ctx, &out) if err != nil { @@ -178,7 +178,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/rm-link", base.String(), link). + err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) if err != nil { return nil, err @@ -194,7 +194,7 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/append-data", p.String()). + err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -211,7 +211,7 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/set-data", p.String()). + err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -237,7 +237,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i var out struct { Changes []change } - if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.ObjectChange, len(out.Changes)) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 8c819121a..639ca2f8e 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -24,7 +24,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } } - if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 0111d626a..f779640f1 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -39,7 +39,7 @@ func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddO return err } - return api.core().request("pin/add", p.String()). + return api.core().Request("pin/add", p.String()). Option("recursive", options.Recursive).Exec(ctx, nil) } @@ -50,7 +50,7 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. } var out pinRefKeyList - err = api.core().request("pin/ls"). + err = api.core().Request("pin/ls"). Option("type", options.Type).Exec(ctx, &out) if err != nil { return nil, err @@ -74,7 +74,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt return err } - return api.core().request("pin/rm", p.String()). + return api.core().Request("pin/rm", p.String()). Option("recursive", options.Recursive). Exec(ctx, nil) } @@ -85,7 +85,7 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o return err } - return api.core().request("pin/update"). + return api.core().Request("pin/update"). Option("unpin", options.Unpin).Exec(ctx, nil) } @@ -116,7 +116,7 @@ func (n *badNode) Err() error { } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { - resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 2ac04b53c..380b933dc 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -18,7 +18,7 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { Strings []string } - if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } @@ -35,7 +35,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +51,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", topic). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -112,7 +112,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, err } - resp, err := api.core().request("pubsub/sub", topic). + resp, err := api.core().Request("pubsub/sub", topic). Option("discover", options.Discover).Send(ctx) if err != nil { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0814debee..0e47df2ab 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -25,11 +25,11 @@ func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs[i] = addr.Encapsulate(pidma).String() } - return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) + return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil) } func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { - return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) + return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil) } type connInfo struct { @@ -75,7 +75,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) } } - err := api.core().request("swarm/peers"). + err := api.core().Request("swarm/peers"). Option("streams", true). Option("latency", true). Exec(ctx, &resp) @@ -116,7 +116,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu var out struct { Addrs map[string][]string } - if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil { return nil, err } res := map[peer.ID][]multiaddr.Multiaddr{} @@ -147,7 +147,7 @@ func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, err Strings []string } - if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil { return nil, err } @@ -167,7 +167,7 @@ func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, er Strings []string } - if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 4f717b129..84609194b 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -36,7 +36,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("add"). + req := api.core().Request("add"). Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). @@ -139,7 +139,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return nil, err } - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-type", options.ResolveChildren). Option("size", options.ResolveChildren). Option("stream", true). From 70158edd5950671a26631788d15204350d174f52 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Apr 2019 00:36:32 -0700 Subject: [PATCH 073/112] feed through context see https://github.com/ipfs/go-ipfs-api/pull/173 all credit to @optman This commit was moved from ipfs/go-ipfs-http-client@7b75a8c577988e4728474a02144bf5bf83260527 --- client/httpapi/request.go | 2 ++ client/httpapi/response.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67..dd5293b7a 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,6 +7,7 @@ import ( ) type Request struct { + Ctx context.Context ApiBase string Command string Args []string @@ -25,6 +26,7 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ + Ctx: ctx, ApiBase: url + "/api/v0", Command: command, Args: args, diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658..29ae38c69 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -95,6 +95,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } + req = req.WithContext(r.Ctx) + // Add any headers that were supplied via the RequestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) From 3b02ea46462d1f87a64e3fe23f73ee0f186b40d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 9 Apr 2019 19:53:07 -0700 Subject: [PATCH 074/112] api: add authenticated transport, and direct connection This commit was moved from ipfs/go-ipfs-http-client@d04afa02b059d2c6a631f23a6923eb57ecb21015 --- client/httpapi/api.go | 36 ++++++++++++++++++++++++++++++++++++ client/httpapi/api_test.go | 25 +++++++++++++++++++++++-- client/httpapi/transport.go | 21 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dad205fd0..c0eacd7cb 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -122,6 +122,42 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } +// NewDirectAPIClient is used to instantiate a HttpApi client +// that connects to an endpoint which leverages additional http paths. +// +// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 +// you should use NewDirectAPIClient. +func NewDirectAPIClient(url string) (*HttpApi, error) { + api := &HttpApi{ + url: url, + httpcli: gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }, + applyGlobal: func(*RequestBuilder) {}, + } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api, nil +} + +// WithAuthorization is used to wrap an instance of HttpApi +// with an authenticated transport, such as JWT +func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { + return &HttpApi{ + url: api.url, + httpcli: gohttp.Client{ + Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), + }, + } +} + func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af..f1b1bf269 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -9,11 +9,11 @@ import ( "sync" "testing" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" - "github.com/ipfs/iptb/testbed/interfaces" + testbedi "github.com/ipfs/iptb/testbed/interfaces" ma "github.com/multiformats/go-multiaddr" ) @@ -211,3 +211,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } + +func TestDirectAPI(t *testing.T) { + type args struct { + url, header, value string + } + tests := []struct { + name string + args args + }{ + {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api, err := NewDirectAPIClient(tt.args.url) + if err != nil { + t.Fatal(err) + } + api = api.WithAuthorization(tt.args.header, tt.args.value) + }) + } +} diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go new file mode 100644 index 000000000..3fb0e6e00 --- /dev/null +++ b/client/httpapi/transport.go @@ -0,0 +1,21 @@ +package httpapi + +import "net/http" + +type transport struct { + header, value string + httptr http.RoundTripper +} + +func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { + return &transport{ + header: header, + value: value, + httptr: tr, + } +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set(t.header, t.value) + return t.httptr.RoundTrip(req) +} From 3b67615201cd38aceb6de61e87da1552be7689fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 26 Apr 2019 20:30:55 +0200 Subject: [PATCH 075/112] gomod: update iface with path refactor This commit was moved from ipfs/go-ipfs-http-client@0bd65a67ee2994966538fae6d034ddb9fac466e6 --- client/httpapi/api_test.go | 7 ++----- client/httpapi/apifile.go | 16 ++++++++-------- client/httpapi/block.go | 11 ++++++----- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 17 +++++++++-------- client/httpapi/object.go | 35 ++++++++++++++++++----------------- client/httpapi/path.go | 18 +++++++++--------- client/httpapi/pin.go | 17 +++++++++-------- client/httpapi/unixfs.go | 9 +++++---- 11 files changed, 75 insertions(+), 73 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af..b1bae5565 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" @@ -171,11 +172,7 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } // empty node is pinned even with --empty-repo, we don't want that - emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - errs <- err - return - } + emptyNode := path.New("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { errs <- err return diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 6e3433a51..afd9934e6 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -10,12 +10,12 @@ import ( "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" unixfs "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k -func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { +func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error p, err = api.core().ResolvePath(ctx, p) @@ -48,7 +48,7 @@ type apiFile struct { ctx context.Context core *HttpApi size int64 - path iface.Path + path path.Path r *Response at int64 @@ -114,7 +114,7 @@ func (f *apiFile) Size() (int64, error) { return f.size, nil } -func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), @@ -177,13 +177,13 @@ func (it *apiIter) Next() bool { switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: - it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getDir(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } case unixfs.TFile: - it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getFile(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false @@ -203,7 +203,7 @@ type apiDir struct { ctx context.Context core *UnixfsAPI size int64 - path iface.Path + path path.Path dec *json.Decoder } @@ -224,7 +224,7 @@ func (d *apiDir) Entries() files.DirIterator { } } -func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 3389f4c31..640f186f5 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -26,8 +27,8 @@ func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) Path() iface.ResolvedPath { - return iface.IpldPath(s.cid) +func (s *blockStat) Path() path.Resolved { + return path.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return &out, nil } -func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { +func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { options, err := caopts.BlockRmOptions(opts...) if err != nil { return err @@ -105,7 +106,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR return nil } -func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { +func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 669b5f893..15d9a9c62 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -9,8 +9,8 @@ import ( "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type httpNodeAdder HttpApi @@ -18,7 +18,7 @@ type HttpDagServ httpNodeAdder type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) + r, err := api.core().Block().Get(ctx, path.IpldPath(c)) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (api *HttpDagServ) Pinning() format.NodeAdder { } func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { - return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? + return api.core().Block().Rm(ctx, path.IpldPath(c)) //TODO: should we force rm? } func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index f7417cb7f..e12caa1c5 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" @@ -37,7 +37,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, } } -func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -93,7 +93,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return res, nil } -func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { +func (api *DhtAPI) Provide(ctx context.Context, p path.Path, opts ...caopts.DhtProvideOption) error { options, err := caopts.DhtProvideOptions(opts...) if err != nil { return err diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 1571a3350..9a47e2ed0 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" ) @@ -22,9 +23,8 @@ func (k *keyOutput) Name() string { return k.JName } -func (k *keyOutput) Path() iface.Path { - p, _ := iface.ParsePath("/ipns/" + k.Id) - return p +func (k *keyOutput) Path() path.Path { + return path.New("/ipns/" + k.Id) } func (k *keyOutput) ID() peer.ID { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 0641d1a22..47227a2ab 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/options/namesys" + "github.com/ipfs/interface-go-ipfs-core/path" ) type NameAPI HttpApi @@ -17,18 +18,18 @@ type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` - path iface.Path + path path.Path } func (e *ipnsEntry) Name() string { return e.JName } -func (e *ipnsEntry) Value() iface.Path { +func (e *ipnsEntry) Value() path.Path { return e.path } -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { +func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { options, err := caopts.NamePublishOptions(opts...) if err != nil { return nil, err @@ -48,8 +49,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - out.path, err = iface.ParsePath(out.JValue) - return &out, err + out.path = path.New(out.JValue) + return &out, out.path.IsValid() } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { @@ -93,7 +94,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name } var ires iface.IpnsResult if err == nil { - ires.Path, err = iface.ParsePath(out.Path) + ires.Path = path.New(out.Path) } select { @@ -109,7 +110,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return res, nil } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, err } - return iface.ParsePath(out.Path) + return path.New(out.Path), nil } func (api *NameAPI) core() *HttpApi { diff --git a/client/httpapi/object.go b/client/httpapi/object.go index a1ee486b3..280b2490f 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,6 +14,7 @@ import ( ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi @@ -41,7 +42,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return n, nil } -func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (path.Resolved, error) { options, err := caopts.ObjectPutOptions(opts...) if err != nil { return nil, err @@ -63,10 +64,10 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *ObjectAPI) Data(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err @@ -98,7 +99,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p path.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -126,7 +127,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er return res, nil } -func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { +func (api *ObjectAPI) Stat(ctx context.Context, p path.Path) (*iface.ObjectStat, error) { var out struct { Hash string NumLinks int @@ -154,7 +155,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat }, nil } -func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.Resolved, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return nil, err @@ -173,10 +174,10 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { +func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) @@ -189,10 +190,10 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AppendData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). @@ -206,10 +207,10 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) SetData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). @@ -223,7 +224,7 @@ func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (i return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type change struct { @@ -233,7 +234,7 @@ type change struct { After cid.Cid } -func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { +func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) { var out struct { Changes []change } @@ -247,10 +248,10 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i Path: ch.Path, } if ch.Before != cid.Undef { - res[i].Before = iface.IpfsPath(ch.Before) + res[i].Before = path.IpfsPath(ch.Before) } if ch.After != cid.Undef { - res[i].After = iface.IpfsPath(ch.After) + res[i].After = path.IpfsPath(ch.After) } } return res, nil diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 639ca2f8e..619dd0e0d 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -6,10 +6,10 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) -func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { +func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { var out struct { Cid cid.Cid RemPath string @@ -18,31 +18,31 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help var err error - if path.Namespace() == "ipns" { - if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + if p.Namespace() == "ipns" { + if p, err = api.Name().Resolve(ctx, p.String()); err != nil { return nil, err } } - if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil { return nil, err } // TODO: - ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+p.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } - root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + root, err := cid.Parse(ipfspath.Path(p.String()).Segments()[1]) if err != nil { return nil, err } - return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil + return path.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { rp, err := api.ResolvePath(ctx, p) if err != nil { return nil, err diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f779640f1..ec0fc91a8 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) @@ -21,11 +22,11 @@ type pinRefKeyList struct { } type pin struct { - path iface.ResolvedPath + path path.Resolved typ string } -func (p *pin) Path() iface.ResolvedPath { +func (p *pin) Path() path.Resolved { return p.path } @@ -33,7 +34,7 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { options, err := caopts.PinAddOptions(opts...) if err != nil { return err @@ -62,13 +63,13 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. if err != nil { return nil, err } - pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) } return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { +func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { options, err := caopts.PinRmOptions(opts...) if err != nil { return err @@ -79,7 +80,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt Exec(ctx, nil) } -func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { +func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { options, err := caopts.PinUpdateOptions(opts...) if err != nil { return err @@ -107,8 +108,8 @@ type badNode struct { cid cid.Cid } -func (n *badNode) Path() iface.ResolvedPath { - return iface.IpldPath(n.cid) +func (n *badNode) Path() path.Resolved { + return path.IpldPath(n.cid) } func (n *badNode) Err() error { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 84609194b..5e27fb036 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,6 +13,7 @@ import ( unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -25,7 +26,7 @@ type addEvent struct { type UnixfsAPI HttpApi -func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.Resolved, error) { options, _, err := caopts.UnixfsAddOptions(opts...) if err != nil { return nil, err @@ -98,7 +99,7 @@ loop: return nil, err } - ifevt.Path = iface.IpfsPath(c) + ifevt.Path = path.IpfsPath(c) } select { @@ -114,7 +115,7 @@ loop: return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type lsLink struct { @@ -133,7 +134,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err From 11a990c7b12db59d9d03cbd9b28c2b8c75abfa26 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:13:59 -0700 Subject: [PATCH 076/112] api: expose headers via HttpApi, copy headers during request build This commit was moved from ipfs/go-ipfs-http-client@033befdef3f06a4a80fb9cf7c3afecc1a383700c --- client/httpapi/api.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c0eacd7cb..712801d53 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" gohttp "net/http" "os" "path" @@ -32,9 +33,9 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // For interface docs see // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { - url string - httpcli gohttp.Client - + url string + httpcli gohttp.Client + Headers http.Header applyGlobal func(*RequestBuilder) } @@ -175,10 +176,17 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { + var headers map[string]string + if api.Headers != nil { + for k := range api.Headers { + headers[k] = api.Headers.Get(k) + } + } return &RequestBuilder{ command: command, args: args, shell: api, + headers: headers, } } From d8c286bb15e628dbe7ddd2370c77ef02a5dc16b2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:15 -0700 Subject: [PATCH 077/112] api: remove WithAuthorization, DirectAPI client, instantiate header map This commit was moved from ipfs/go-ipfs-http-client@9c7d495b03db4d130a8fd87217093beb33e6fdb7 --- client/httpapi/api.go | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 712801d53..ffcc85f2b 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -112,6 +112,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -123,20 +124,11 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } -// NewDirectAPIClient is used to instantiate a HttpApi client -// that connects to an endpoint which leverages additional http paths. -// -// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 -// you should use NewDirectAPIClient. -func NewDirectAPIClient(url string) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ - url: url, - httpcli: gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - }, - }, + url: url, + httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -144,21 +136,9 @@ func NewDirectAPIClient(url string) (*HttpApi, error) { api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { return fmt.Errorf("unexpected redirect") } - return api, nil } -// WithAuthorization is used to wrap an instance of HttpApi -// with an authenticated transport, such as JWT -func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { - return &HttpApi{ - url: api.url, - httpcli: gohttp.Client{ - Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), - }, - } -} - func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { @@ -176,7 +156,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { - var headers map[string]string + headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) From 89600e62ce160cf5f3aa2a752b3806a273d1d142 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:37 -0700 Subject: [PATCH 078/112] api: add wip api test This commit was moved from ipfs/go-ipfs-http-client@2b48dd36ad170149bd4c62a4194587bf7c704f7e --- client/httpapi/api_test.go | 62 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index f1b1bf269..7b519ce7d 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,11 +3,13 @@ package httpapi import ( "context" "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" "sync" "testing" + "time" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +214,51 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func TestDirectAPI(t *testing.T) { - type args struct { - url, header, value string +func Test_NewURLApiWithClient(t *testing.T) { + t.Skip() + var ( + url = "127.0.0.1:65501" + headerToTest = "Test-Header" + expectedHeaderValue = "thisisaheadertest" + ) + server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) + if err != nil { + t.Fatal(err) } - tests := []struct { - name string - args args - }{ - {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + defer server.Close() + go func() { + server.ListenAndServe() + }() + time.Sleep(time.Second * 2) + api, err := NewURLApiWithClient(url, &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }) + if err != nil { + t.Fatal(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - api, err := NewDirectAPIClient(tt.args.url) - if err != nil { - t.Fatal(err) - } - api = api.WithAuthorization(tt.args.header, tt.args.value) - }) + api.Headers.Set(headerToTest, expectedHeaderValue) + if _, err := api.Pin().Ls(context.Background()); err != nil { + t.Fatal(err) } } + +/// testHTTPServer spins up a test go http server +// used to check headers +func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { + r := http.NewServeMux() + r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }) + return &http.Server{ + Handler: r, + Addr: url, + }, nil +} From efe3b6054e8f62191e74e6664adec06c306c06aa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:00:48 -0700 Subject: [PATCH 079/112] api: call NewURLApiWithClient from NewApiWithClient This commit was moved from ipfs/go-ipfs-http-client@aea6890f36001f15d24a4b373979bb1b9bf81483 --- client/httpapi/api.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ffcc85f2b..eb99230eb 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -109,19 +109,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { } } - api := &HttpApi{ - url: url, - httpcli: *c, - Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, - } - - // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { - return fmt.Errorf("unexpected redirect") - } - - return api, nil + return NewURLApiWithClient(url, c) } func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { From fd5010bd2a852ea8fa3a2b08e199312fb3b71097 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:13:47 -0700 Subject: [PATCH 080/112] api: cleanup header and NewURLApiWithClient test This commit was moved from ipfs/go-ipfs-http-client@b8b55cb89a5dd830915f83e43e8afa7a01597905 --- client/httpapi/api_test.go | 46 +++++++++++++------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 52fa67d0e..fadd9663e 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -5,13 +5,14 @@ import ( "io/ioutil" "net/http" gohttp "net/http" + "net/http/httptest" "os" "strconv" "sync" "testing" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +213,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func Test_NewURLApiWithClient(t *testing.T) { - t.Skip() +func Test_NewURLApiWithClient_With_Headers(t *testing.T) { var ( - url = "127.0.0.1:65501" headerToTest = "Test-Header" expectedHeaderValue = "thisisaheadertest" ) - server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) - if err != nil { - t.Fatal(err) - } - defer server.Close() - go func() { - server.ListenAndServe() - }() + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }), + ) + defer ts.Close() time.Sleep(time.Second * 2) - api, err := NewURLApiWithClient(url, &http.Client{ + api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, @@ -242,21 +244,3 @@ func Test_NewURLApiWithClient(t *testing.T) { t.Fatal(err) } } - -/// testHTTPServer spins up a test go http server -// used to check headers -func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { - r := http.NewServeMux() - r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { - val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { - w.WriteHeader(400) - return - } - w.WriteHeader(200) - }) - return &http.Server{ - Handler: r, - Addr: url, - }, nil -} From daf1b72809a3d1af819d7ff44aabcc6bc6471fe8 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 17:19:35 -0700 Subject: [PATCH 081/112] api: fix failing test This commit was moved from ipfs/go-ipfs-http-client@ea507216d821455c351440ba33775f564ea18ea8 --- client/httpapi/api_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fadd9663e..5f012b476 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "strconv" + "strings" "sync" "testing" "time" @@ -221,15 +222,14 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { + if val != expectedHeaderValue { w.WriteHeader(400) return } - w.WriteHeader(200) + http.ServeContent(w, r, "", time.Now(), strings.NewReader("test")) }), ) defer ts.Close() - time.Sleep(time.Second * 2) api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -240,7 +240,7 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } api.Headers.Set(headerToTest, expectedHeaderValue) - if _, err := api.Pin().Ls(context.Background()); err != nil { + if err := api.Pin().Rm(context.Background(), path.New("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")); err != nil { t.Fatal(err) } } From 8b9e189169b51c46956a18e08f9dc7bc668401b4 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 19:14:15 -0700 Subject: [PATCH 082/112] remove unused transport.go file This commit was moved from ipfs/go-ipfs-http-client@320130421f4727d72d68a3c6a5c0e633e1dabc38 --- client/httpapi/transport.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go deleted file mode 100644 index 3fb0e6e00..000000000 --- a/client/httpapi/transport.go +++ /dev/null @@ -1,21 +0,0 @@ -package httpapi - -import "net/http" - -type transport struct { - header, value string - httptr http.RoundTripper -} - -func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { - return &transport{ - header: header, - value: value, - httptr: tr, - } -} - -func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Set(t.header, t.value) - return t.httptr.RoundTrip(req) -} From 8d953a7359c488cde5299ca704594bd8e1349ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 10 May 2019 15:38:20 +0200 Subject: [PATCH 083/112] Use interface for RequestBuilder This commit was moved from ipfs/go-ipfs-http-client@0534b7ca8319329459c505a546f3025862a735f5 --- client/httpapi/api.go | 10 ++++----- client/httpapi/requestbuilder.go | 36 ++++++++++++++++++++++---------- client/httpapi/response.go | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index eb99230eb..5c2eb76d6 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -36,7 +36,7 @@ type HttpApi struct { url string httpcli gohttp.Client Headers http.Header - applyGlobal func(*RequestBuilder) + applyGlobal func(*requestBuilder) } // NewLocalApi tries to construct new HttpApi instance communicating with local @@ -117,7 +117,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { url: url, httpcli: *c, Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, + applyGlobal: func(*requestBuilder) {}, } // We don't support redirects. @@ -134,7 +134,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } subApi := *api - subApi.applyGlobal = func(req *RequestBuilder) { + subApi.applyGlobal = func(req *requestBuilder) { if options.Offline { req.Option("offline", options.Offline) } @@ -143,14 +143,14 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) RequestBuilder { headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) } } - return &RequestBuilder{ + return &requestBuilder{ command: command, args: args, shell: api, diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 2ffed7a0a..7012a8935 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -12,8 +12,20 @@ import ( "github.com/ipfs/go-ipfs-files" ) -// RequestBuilder is an IPFS commands request builder. -type RequestBuilder struct { +type RequestBuilder interface { + Arguments(args ...string) RequestBuilder + BodyString(body string) RequestBuilder + BodyBytes(body []byte) RequestBuilder + Body(body io.Reader) RequestBuilder + FileBody(body io.Reader) RequestBuilder + Option(key string, value interface{}) RequestBuilder + Header(name, value string) RequestBuilder + Send(ctx context.Context) (*Response, error) + Exec(ctx context.Context, res interface{}) error +} + +// requestBuilder is an IPFS commands request builder. +type requestBuilder struct { command string args []string opts map[string]string @@ -24,29 +36,29 @@ type RequestBuilder struct { } // Arguments adds the arguments to the args. -func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { +func (r *requestBuilder) Arguments(args ...string) RequestBuilder { r.args = append(r.args, args...) return r } // BodyString sets the request body to the given string. -func (r *RequestBuilder) BodyString(body string) *RequestBuilder { +func (r *requestBuilder) BodyString(body string) RequestBuilder { return r.Body(strings.NewReader(body)) } // BodyBytes sets the request body to the given buffer. -func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { +func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder { return r.Body(bytes.NewReader(body)) } // Body sets the request body to the given reader. -func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { +func (r *requestBuilder) Body(body io.Reader) RequestBuilder { r.body = body return r } // FileBody sets the request body to the given reader wrapped into multipartreader. -func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { +func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) @@ -55,7 +67,7 @@ func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { } // Option sets the given option. -func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { +func (r *requestBuilder) Option(key string, value interface{}) RequestBuilder { var s string switch v := value.(type) { case bool: @@ -76,7 +88,7 @@ func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { } // Header sets the given header. -func (r *RequestBuilder) Header(name, value string) *RequestBuilder { +func (r *requestBuilder) Header(name, value string) RequestBuilder { if r.headers == nil { r.headers = make(map[string]string, 1) } @@ -85,7 +97,7 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { } // Send sends the request and return the response. -func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { +func (r *requestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) req := NewRequest(ctx, r.shell.url, r.command, r.args...) @@ -96,7 +108,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { } // Exec sends the request a request and decodes the response. -func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { +func (r *requestBuilder) Exec(ctx context.Context, res interface{}) error { httpRes, err := r.Send(ctx) if err != nil { return err @@ -112,3 +124,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return httpRes.decode(res) } + +var _ RequestBuilder = &requestBuilder{} diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 2d6af49d2..db22a66fa 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -97,7 +97,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { req = req.WithContext(r.Ctx) - // Add any headers that were supplied via the RequestBuilder. + // Add any headers that were supplied via the requestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) } From 1b81f2ef10c40b5ba5a50ae545a62cc050c11813 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 11 Jun 2019 17:41:49 -0700 Subject: [PATCH 084/112] add extended error handling fixes #19 This commit was moved from ipfs/go-ipfs-http-client@8e3552ac1e6ea18ffd5f49918eef578de2b224fb --- client/httpapi/response.go | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index db22a66fa..056b2d68d 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -13,6 +13,48 @@ import ( "os" ) +// Error codes adapted from go-ipfs-cmds. We should find a better solution. + +// ErrorType signfies a category of errors +type ErrorType uint + +// ErrorTypes convey what category of error ocurred +const ( + // ErrNormal is a normal error. The command failed for some reason that's not a bug. + ErrNormal ErrorType = iota + // ErrClient means the client made an invalid request. + ErrClient + // ErrImplementation means there's a bug in the implementation. + ErrImplementation + // ErrRateLimited is returned when the operation has been rate-limited. + ErrRateLimited + // ErrForbidden is returned when the client doesn't have permission to + // perform the requested operation. + ErrForbidden +) + +func (e ErrorType) Error() string { + return e.String() +} + +func (e ErrorType) String() string { + switch e { + case ErrNormal: + return "command failed" + case ErrClient: + return "invalid argument" + case ErrImplementation: + return "internal error" + case ErrRateLimited: + return "rate limited" + case ErrForbidden: + return "request forbidden" + default: + return "unknown error code" + } + +} + type trailerReader struct { resp *http.Response } @@ -77,7 +119,13 @@ func (r *Response) decode(dec interface{}) error { type Error struct { Command string Message string - Code int + Code ErrorType +} + +// Unwrap returns the base error (an ErrorType). Works with go 1.14 error +// helpers. +func (e *Error) Unwrap() error { + return e.Code } func (e *Error) Error() string { @@ -133,11 +181,23 @@ func (r *Request) Send(c *http.Client) (*Response, error) { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = string(out) + + // set special status codes. + switch resp.StatusCode { + case http.StatusNotFound, http.StatusBadRequest: + e.Code = ErrClient + case http.StatusTooManyRequests: + e.Code = ErrRateLimited + case http.StatusForbidden: + e.Code = ErrForbidden + } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) } default: + // This is a server-side bug (probably). + e.Code = ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 2612353d85385a595e889dfa247363a8a4ab02c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 13 Jun 2019 12:00:58 -0700 Subject: [PATCH 085/112] use error from go-ipfs-cmds note: this drops the Command field This commit was moved from ipfs/go-ipfs-http-client@523a26f0a853a2d6779a21d7bad239bf2012833b --- client/httpapi/response.go | 81 ++++++-------------------------------- 1 file changed, 11 insertions(+), 70 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 056b2d68d..95cbf13ec 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,56 +4,19 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" "mime" "net/http" "net/url" "os" + + cmds "github.com/ipfs/go-ipfs-cmds" + cmdhttp "github.com/ipfs/go-ipfs-cmds/http" + files "github.com/ipfs/go-ipfs-files" ) -// Error codes adapted from go-ipfs-cmds. We should find a better solution. - -// ErrorType signfies a category of errors -type ErrorType uint - -// ErrorTypes convey what category of error ocurred -const ( - // ErrNormal is a normal error. The command failed for some reason that's not a bug. - ErrNormal ErrorType = iota - // ErrClient means the client made an invalid request. - ErrClient - // ErrImplementation means there's a bug in the implementation. - ErrImplementation - // ErrRateLimited is returned when the operation has been rate-limited. - ErrRateLimited - // ErrForbidden is returned when the client doesn't have permission to - // perform the requested operation. - ErrForbidden -) - -func (e ErrorType) Error() string { - return e.String() -} - -func (e ErrorType) String() string { - switch e { - case ErrNormal: - return "command failed" - case ErrClient: - return "invalid argument" - case ErrImplementation: - return "internal error" - case ErrRateLimited: - return "rate limited" - case ErrForbidden: - return "request forbidden" - default: - return "unknown error code" - } - -} +type Error = cmds.Error type trailerReader struct { resp *http.Response @@ -62,7 +25,7 @@ type trailerReader struct { func (r *trailerReader) Read(b []byte) (int, error) { n, err := r.resp.Body.Read(b) if err != nil { - if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" { err = errors.New(e) } } @@ -116,26 +79,6 @@ func (r *Response) decode(dec interface{}) error { return err2 } -type Error struct { - Command string - Message string - Code ErrorType -} - -// Unwrap returns the base error (an ErrorType). Works with go 1.14 error -// helpers. -func (e *Error) Unwrap() error { - return e.Code -} - -func (e *Error) Error() string { - var out string - if e.Code != 0 { - out = fmt.Sprintf("%s%d: ", out, e.Code) - } - return out + e.Message -} - func (r *Request) Send(c *http.Client) (*Response, error) { url := r.getURL() req, err := http.NewRequest("POST", url, r.Body) @@ -169,9 +112,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { - e := &Error{ - Command: r.Command, - } + e := new(Error) switch { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" @@ -185,11 +126,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // set special status codes. switch resp.StatusCode { case http.StatusNotFound, http.StatusBadRequest: - e.Code = ErrClient + e.Code = cmds.ErrClient case http.StatusTooManyRequests: - e.Code = ErrRateLimited + e.Code = cmds.ErrRateLimited case http.StatusForbidden: - e.Code = ErrForbidden + e.Code = cmds.ErrForbidden } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { @@ -197,7 +138,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { } default: // This is a server-side bug (probably). - e.Code = ErrImplementation + e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 9345cfa7b83d3fb2fb470fa1e77e7af89087452b Mon Sep 17 00:00:00 2001 From: godcong Date: Sun, 21 Jul 2019 12:33:35 +0800 Subject: [PATCH 086/112] update to consolidated libp2p interface package (#21) and fix parsing of connection latencies This commit was moved from ipfs/go-ipfs-http-client@fd5cce4cbc9d95617ded33849c437a492f5e9d96 --- client/httpapi/dht.go | 31 +++++++++++++++---------------- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/swarm.go | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index e12caa1c5..0a78514f9 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,38 +6,37 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - notif "github.com/libp2p/go-libp2p-routing/notifications" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" ) type DhtAPI HttpApi -func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { var out struct { - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } if resp.Error != nil { - return peerstore.PeerInfo{}, resp.Error + return peer.AddrInfo{}, resp.Error } defer resp.Close() dec := json.NewDecoder(resp.Output) for { if err := dec.Decode(&out); err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } - if out.Type == notif.FinalPeer { + if out.Type == routing.FinalPeer { return out.Responses[0], nil } } } -func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -57,7 +56,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt if resp.Error != nil { return nil, resp.Error } - res := make(chan peerstore.PeerInfo) + res := make(chan peer.AddrInfo) go func() { defer resp.Close() @@ -67,18 +66,18 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt for { var out struct { Extra string - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } - if out.Type == notif.QueryError { + if out.Type == routing.QueryError { return // usually a 'not found' error // todo: handle other errors } - if out.Type == notif.Provider { + if out.Type == routing.Provider { for _, pi := range out.Responses { select { case res <- pi: diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 9a47e2ed0..cf22c6108 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 380b933dc..81ddb5211 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type PubsubAPI HttpApi diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0e47df2ab..10280c39f 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,16 +5,15 @@ import ( "time" "github.com/ipfs/interface-go-ipfs-core" - inet "github.com/libp2p/go-libp2p-net" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - "github.com/libp2p/go-libp2p-protocol" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/multiformats/go-multiaddr" ) type SwarmAPI HttpApi -func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { +func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) if err != nil { return err @@ -37,7 +36,7 @@ type connInfo struct { peer peer.ID latency time.Duration muxer string - direction inet.Direction + direction network.Direction streams []protocol.ID } @@ -49,7 +48,7 @@ func (c *connInfo) Address() multiaddr.Multiaddr { return c.addr } -func (c *connInfo) Direction() inet.Direction { +func (c *connInfo) Direction() network.Direction { return c.direction } @@ -66,9 +65,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) Peers []struct { Addr string Peer string - Latency time.Duration + Latency string Muxer string - Direction inet.Direction + Direction network.Direction Streams []struct { Protocol string } @@ -85,8 +84,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { + latency, _ := time.ParseDuration(conn.Latency) out := &connInfo{ - latency: conn.Latency, + latency: latency, muxer: conn.Muxer, direction: conn.Direction, } From be8f80237d4ba8ed8d2ef278180b2b3d446c60f0 Mon Sep 17 00:00:00 2001 From: godcong Date: Tue, 13 Aug 2019 07:16:47 +0800 Subject: [PATCH 087/112] fix path miss in windows/add a new api create function (#23) * fix path was not always correct in windows * add new api create function for if the port was different with api file. * Update api.go remove no need commit This commit was moved from ipfs/go-ipfs-http-client@8c87debb1233f2af7a0cd102a934b630d8a12147 --- client/httpapi/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 5c2eb76d6..ec16c4b2d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -7,12 +7,12 @@ import ( "net/http" gohttp "net/http" "os" - "path" + "path/filepath" "strings" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - homedir "github.com/mitchellh/go-homedir" + "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -73,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return nil, err } - apiFile := path.Join(baseDir, DefaultApiFile) + apiFile := filepath.Join(baseDir, DefaultApiFile) api, err := ioutil.ReadFile(apiFile) if err != nil { From b6b2ae355bf86c4859f234240b44b35656685ece Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 30 Aug 2019 18:44:24 -0700 Subject: [PATCH 088/112] doc: mark this package experimental instead of WIP (#30) We're at a point where this package works pretty well and users should consider using it. This commit was moved from ipfs/go-ipfs-http-client@3e8506bbecd9b77ae30d04c7b997d9d612654beb --- client/httpapi/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index dd09142e3..e5ab1c612 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,7 +8,10 @@ > IPFS CoreAPI implementation using HTTP API -This project is WIP, use https://github.com/ipfs/go-ipfs-api for now +This package is experimental and subject to change. If you need to depend on +something less likely to change, please use +[go-ipfs-api](https://github.com/ipfs/go-ipfs-api). If you'd like the latest and +greatest features, please use _this_ package. ## Documentation From d48f8fce64d6496a18b442c74021829554151823 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 10 Sep 2019 18:07:48 -0700 Subject: [PATCH 089/112] file: implement ReadAt This commit was moved from ipfs/go-ipfs-http-client@88a9b615205816ea20ea38fbee170b3256e51b94 --- client/httpapi/apifile.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index afd9934e6..ec2f7588d 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -82,6 +82,25 @@ func (f *apiFile) Read(p []byte) (int, error) { return n, err } +func (f *apiFile) ReadAt(p []byte, off int64) (int, error) { + // Always make a new request. This method should be parallel-safe. + resp, err := f.core.Request("cat", f.path.String()). + Option("offset", off).Option("length", len(p)).Send(f.ctx) + if err != nil { + return 0, err + } + if resp.Error != nil { + return 0, resp.Error + } + defer resp.Output.Close() + + n, err := io.ReadFull(resp.Output, p) + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + return n, err +} + func (f *apiFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekEnd: From 8f9306304efdc60147eccbb49380b83e3c7cb4ea Mon Sep 17 00:00:00 2001 From: Joel Gustafson Date: Sat, 7 Dec 2019 01:08:48 -0500 Subject: [PATCH 090/112] Update pin.go This commit was moved from ipfs/go-ipfs-http-client@31b4e60d961ada25cb7b40317b18011b284436a7 --- client/httpapi/pin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ec0fc91a8..6026c71b3 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -86,7 +86,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt return err } - return api.core().Request("pin/update"). + return api.core().Request("pin/update", from.String(), to.String()). Option("unpin", options.Unpin).Exec(ctx, nil) } From d9004a9061070263dcc01976334836acb9493530 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 3 May 2020 12:04:04 -0700 Subject: [PATCH 091/112] IDB58Decode -> Decode Allow Non-RSA keys to be decoded by the HTTP client This commit was moved from ipfs/go-ipfs-http-client@b0549d86231d69a933b0e4b771c350cf1bf5c27b --- client/httpapi/key.go | 12 ++++++------ client/httpapi/pubsub.go | 4 ++-- client/httpapi/swarm.go | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index cf22c6108..78b67517c 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/peer" @@ -45,7 +45,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -69,7 +69,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - id.pid, err = peer.IDB58Decode(id.Id) + id.pid, err = peer.Decode(id.Id) return id, out.Overwrite, err } @@ -82,7 +82,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { var err error - k.pid, err = peer.IDB58Decode(k.Id) + k.pid, err = peer.Decode(k.Id) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var err error out := keyOutput{JName: "self", Id: id.ID} - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -114,7 +114,7 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { } var err error - out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + out.Keys[0].pid, err = peer.Decode(out.Keys[0].Id) return &out.Keys[0], err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 81ddb5211..da7c59ef1 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,7 +6,7 @@ import ( "encoding/json" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" ) @@ -41,7 +41,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio res := make([]peer.ID, len(out.Strings)) for i, sid := range out.Strings { - id, err := peer.IDB58Decode(sid) + id, err := peer.Decode(sid) if err != nil { return nil, err } diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 10280c39f..1cb0d91df 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -91,7 +91,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) direction: conn.Direction, } - out.peer, err = peer.IDB58Decode(conn.Peer) + out.peer, err = peer.Decode(conn.Peer) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu addrs[i] = a } - pid, err := peer.IDB58Decode(spid) + pid, err := peer.Decode(spid) if err != nil { return nil, err } From 7ebef96ba12bfdfc550412748ee16eadcaffd25d Mon Sep 17 00:00:00 2001 From: Bryan Stenson Date: Tue, 12 May 2020 20:09:16 +0000 Subject: [PATCH 092/112] update contributing link This commit was moved from ipfs/go-ipfs-http-client@ef2a556e19e8d9ebafcb24b3574724c6d7718bc8 --- client/httpapi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index e5ab1c612..376e6590e 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -25,7 +25,7 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ### Want to hack on IPFS? -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## License From 247cb05fbc4fa3863542ecd9e0e5109c927790c6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 31 Jul 2020 10:20:59 -0700 Subject: [PATCH 093/112] Bump interface-go-ipfs-core version (#117) * fix issues #26 Implement an IsPinned function * fix issues #113 Implement interface v0.3.0 * ci: update to use go-ipfs v0.6.0 * ci: bump go version Co-authored-by: godcong Co-authored-by: Adin Schmahmann This commit was moved from ipfs/go-ipfs-http-client@4e8a6c8b981f808062caa4ef7481babc0a39f298 --- client/httpapi/pin.go | 54 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 6026c71b3..bd3e5423b 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,9 +3,10 @@ package httpapi import ( "context" "encoding/json" + "strings" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" @@ -24,6 +25,11 @@ type pinRefKeyList struct { type pin struct { path path.Resolved typ string + err error +} + +func (p *pin) Err() error { + return p.err } func (p *pin) Path() path.Resolved { @@ -44,7 +50,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp Option("recursive", options.Recursive).Exec(ctx, nil) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { options, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err @@ -57,16 +63,46 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return nil, err } - pins := make([]iface.Pin, 0, len(out.Keys)) - for hash, p := range out.Keys { - c, err := cid.Parse(hash) - if err != nil { - return nil, err + pins := make(chan iface.Pin) + go func(ch chan<- iface.Pin) { + defer close(ch) + for hash, p := range out.Keys { + c, e := cid.Parse(hash) + if e != nil { + ch <- &pin{typ: p.Type, err: e} + return + } + ch <- &pin{typ: p.Type, path: path.IpldPath(c), err: e} } - pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) + }(pins) + return pins, nil +} + +// IsPinned returns whether or not the given cid is pinned +// and an explanation of why its pinned +func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { + options, err := caopts.PinIsPinnedOptions(opts...) + if err != nil { + return "", false, err + } + var out pinRefKeyList + err = api.core().Request("pin/ls"). + Option("type", options.WithType). + Option("arg", p.String()). + Exec(ctx, &out) + if err != nil { + // TODO: This error-type discrimination based on sub-string matching is brittle. + // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 + if strings.Index(err.Error(), "is not pinned") != -1 { + return "", false, nil + } + return "", false, err } - return pins, nil + for _, obj := range out.Keys { + return obj.Type, true, nil + } + return "", false, errors.New("http api returned no error and no results") } func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { From ea2a35686db523b544245cbaea5592021d0cd163 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 15 May 2021 18:21:24 -0700 Subject: [PATCH 094/112] fix staticcheck This commit was moved from ipfs/go-ipfs-http-client@4461a0bb952ce0a05a5bab9728cc64b658375068 --- client/httpapi/api.go | 17 ++++++++--------- client/httpapi/api_test.go | 7 +++---- client/httpapi/object.go | 3 +-- client/httpapi/pin.go | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ec16c4b2d..f589e4267 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/http" - gohttp "net/http" "os" "path/filepath" "strings" @@ -14,7 +13,7 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) const ( @@ -34,7 +33,7 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string - httpcli gohttp.Client + httpcli http.Client Headers http.Header applyGlobal func(*requestBuilder) } @@ -85,9 +84,9 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { // NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, }, } @@ -96,7 +95,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { } // NewApiWithClient constructs HttpApi with specified endpoint and custom http client -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { +func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { return nil, err @@ -112,7 +111,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return NewURLApiWithClient(url, c) } -func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, @@ -121,7 +120,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { } // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("unexpected redirect") } return api, nil diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5f012b476..35a57d2c0 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,7 +4,6 @@ import ( "context" "io/ioutil" "net/http" - gohttp "net/http" "net/http/httptest" "os" "strconv" @@ -163,9 +162,9 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) return } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, DisableCompression: true, }, diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 280b2490f..6ec7f5503 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - dag "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -32,7 +31,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( var n ipld.Node switch options.Type { case "empty": - n = new(dag.ProtoNode) + n = new(merkledag.ProtoNode) case "unixfs-dir": n = ft.EmptyDirNode() default: diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index bd3e5423b..13de2d389 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -93,7 +93,7 @@ func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.Pin if err != nil { // TODO: This error-type discrimination based on sub-string matching is brittle. // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 - if strings.Index(err.Error(), "is not pinned") != -1 { + if strings.Contains(err.Error(), "is not pinned") { return "", false, nil } return "", false, err From 5322cf89e45197afb74cc36104ee581ca179c1d6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 Nov 2021 22:24:27 +0100 Subject: [PATCH 095/112] feat: pubsub http rpc with multibase (#151) * feat: pubsub http rpc with multibase This updates HTTP RPC wire format to one from https://github.com/ipfs/go-ipfs/pull/8183 * chore: use updated go-ipfs * chore: switch ci to go-ipfs master This commit was moved from ipfs/go-ipfs-http-client@c832fc0ce1d8235c830fecd0dd44dda04a9fba37 --- client/httpapi/README.md | 6 ++-- client/httpapi/pubsub.go | 78 +++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index 376e6590e..d3e14ac3f 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -1,8 +1,8 @@ # go-ipfs-http-api -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](https://protocol.ai) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.io/) +[![](https://img.shields.io/badge/matrix-%23ipfs-blue.svg?style=flat-square)](https://app.element.io/#/room/#ipfs:matrix.org) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index da7c59ef1..72f592376 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -9,6 +9,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" + mbase "github.com/multiformats/go-multibase" ) type PubsubAPI HttpApi @@ -21,8 +22,15 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } - - return out.Strings, nil + topics := make([]string, len(out.Strings)) + for n, mb := range out.Strings { + _, topic, err := mbase.Decode(mb) + if err != nil { + return nil, err + } + topics[n] = string(topic) + } + return topics, nil } func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { @@ -35,7 +43,11 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + var optionalTopic string + if len(options.Topic) > 0 { + optionalTopic = toMultibase([]byte(options.Topic)) + } + if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +63,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().Request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", toMultibase([]byte(topic))). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -64,13 +76,18 @@ type pubsubSub struct { } type pubsubMessage struct { - JFrom []byte `json:"from,omitempty"` - JData []byte `json:"data,omitempty"` - JSeqno []byte `json:"seqno,omitempty"` + JFrom string `json:"from,omitempty"` + JData string `json:"data,omitempty"` + JSeqno string `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` - from peer.ID - err error + // real values after unpacking from text/multibase envelopes + from peer.ID + data []byte + seqno []byte + topics []string + + err error } func (msg *pubsubMessage) From() peer.ID { @@ -78,15 +95,17 @@ func (msg *pubsubMessage) From() peer.ID { } func (msg *pubsubMessage) Data() []byte { - return msg.JData + return msg.data } func (msg *pubsubMessage) Seq() []byte { - return msg.JSeqno + return msg.seqno } +// TODO: do we want to keep this interface as []string, +// or change to more correct [][]byte? func (msg *pubsubMessage) Topics() []string { - return msg.JTopicIDs + return msg.topics } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { @@ -98,22 +117,41 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if msg.err != nil { return nil, msg.err } + // unpack values from text/multibase envelopes var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err + msg.from, err = peer.Decode(msg.JFrom) + if err != nil { + return nil, err + } + _, msg.data, err = mbase.Decode(msg.JData) + if err != nil { + return nil, err + } + _, msg.seqno, err = mbase.Decode(msg.JSeqno) + if err != nil { + return nil, err + } + for _, mbt := range msg.JTopicIDs { + _, topic, err := mbase.Decode(mbt) + if err != nil { + return nil, err + } + msg.topics = append(msg.topics, string(topic)) + } + return &msg, nil case <-ctx.Done(): return nil, ctx.Err() } } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + /* right now we have no options (discover got deprecated) options, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } - - resp, err := api.core().Request("pubsub/sub", topic). - Option("discover", options.Discover).Send(ctx) + */ + resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx) if err != nil { return nil, err @@ -168,3 +206,9 @@ func (s *pubsubSub) Close() error { func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } + +// Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body) +func toMultibase(data []byte) string { + mb, _ := mbase.Encode(mbase.Base64url, data) + return mb +} From b52c80ca4da5730f47674946025c9490fcd50e1f Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 14 Jan 2022 16:16:26 +0100 Subject: [PATCH 096/112] skip TestHttpApi on Windows This commit was moved from ipfs/go-ipfs-http-client@a41c7956c0da2a4c51e15794b22dea09395cc7ef --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 35a57d2c0..d5f27f0ec 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "runtime" "strconv" "strings" "sync" @@ -207,6 +208,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } func TestHttpApi(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping due to #142") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() From b85aa9639f861d659b49e0396b409398462226a2 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 27 Mar 2022 14:11:47 +0200 Subject: [PATCH 097/112] fix: make Block().* return correct ABI based ipld.ErrNotFound errors This commit was moved from ipfs/go-ipfs-http-client@4f5f8e9b144d1f38e0fc37c5080845611dcbcf1e --- client/httpapi/abyfy_errors.go | 104 ++++++++++++++++++++++++++++ client/httpapi/abyfy_errors_test.go | 60 ++++++++++++++++ client/httpapi/block.go | 11 +-- 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 client/httpapi/abyfy_errors.go create mode 100644 client/httpapi/abyfy_errors_test.go diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/abyfy_errors.go new file mode 100644 index 000000000..e4f4afb7c --- /dev/null +++ b/client/httpapi/abyfy_errors.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "errors" + "strings" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +type prePostWrappedNotFoundError struct { + pre string + post string + + wrapped ipld.ErrNotFound +} + +func (e prePostWrappedNotFoundError) String() string { + return e.Error() +} + +func (e prePostWrappedNotFoundError) Error() string { + return e.pre + e.wrapped.Error() + e.post +} + +func (e prePostWrappedNotFoundError) Unwrap() error { + return e.wrapped +} + +func abyfyIpldNotFoundFallbackToMSG(msg string) error { + err, handled := abyfyIpldNotFound(msg) + if handled { + return err + } + + return errors.New(msg) +} + +func abyfyIpldNotFoundFallbackToError(msg error) error { + err, handled := abyfyIpldNotFound(msg.Error()) + if handled { + return err + } + + return msg +} + +// This file handle parsing and returning the correct ABI based errors from error messages +//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type +func abyfyIpldNotFound(msg string) (error, bool) { + if msg == "" { + return nil, true // Fast path + } + + // The patern we search for is: + // node not found (fallback) + // or + // CID not found (here we parse the CID) + notFoundIndex := strings.LastIndex(msg, " not found") + + if notFoundIndex == -1 { + // Unknown, ot found not found + return nil, false + } + + preNotFound := msg[:notFoundIndex] + + var c cid.Cid + var preIndex int + if strings.HasSuffix(preNotFound, "node") { + // Fallback case + c = cid.Undef + preIndex = notFoundIndex - len("node") + } else { + // Assume that CIDs does not include whitespace to pull out the CID + preIndex = strings.LastIndexByte(preNotFound, ' ') + // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space + preIndex++ + var err error + c, err = cid.Decode(preNotFound[preIndex:]) + if err != nil { + // Unknown + return nil, false + } + } + + postIndex := notFoundIndex + len(" not found") + + err := ipld.ErrNotFound{Cid: c} + + pre := msg[:preIndex] + post := msg[postIndex:] + + if len(pre) > 0 || len(post) > 0 { + // We have some text to wrap arround the ErrNotFound one + return prePostWrappedNotFoundError{ + pre: pre, + post: post, + wrapped: err, + }, true + } + + return err, true +} diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/abyfy_errors_test.go new file mode 100644 index 000000000..fe1cf0707 --- /dev/null +++ b/client/httpapi/abyfy_errors_test.go @@ -0,0 +1,60 @@ +package httpapi + +import ( + "errors" + "fmt" + "testing" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} + +func doAbyfyIpldNotFoundTest(t *testing.T, original error) { + originalMsg := original.Error() + + rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + + rebuiltMsg := rebuilt.Error() + + if originalMsg != rebuiltMsg { + t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg) + } + + originalNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(original) + if originalNotFound != rebuiltNotFound { + t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + } +} + +func TestAbyfyIpldNotFound(t *testing.T) { + if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { + t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + for _, err := range [...]error{ + errors.New("file not found"), + errors.New(" not found"), + errors.New("Bad_CID not found"), + errors.New("network connection timeout"), + ipld.ErrNotFound{Cid: cid.Undef}, + ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, + ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doAbyfyIpldNotFoundTest(t, err) + } + } +} diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 640f186f5..e78ba2281 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -3,7 +3,6 @@ package httpapi import ( "bytes" "context" - "errors" "fmt" "io" @@ -67,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, resp.Error + return nil, abyfyIpldNotFoundFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -99,18 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - if removedBlock.Error != "" { - return errors.New(removedBlock.Error) - } - - return nil + return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, err + return nil, abyfyIpldNotFoundFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { From 5a8e2b77c0e4361a0dd6e57a7f4129adba2f4d8a Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 30 Mar 2022 04:32:12 +0200 Subject: [PATCH 098/112] chore: rename abyfyIpldErrNotFound to parseIPLDErrNotFound This commit was moved from ipfs/go-ipfs-http-client@7aa002992970058e37bb82f9837a442227562d33 --- client/httpapi/block.go | 6 +++--- client/httpapi/{abyfy_errors.go => errors.go} | 10 +++++----- .../httpapi/{abyfy_errors_test.go => errors_test.go} | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) rename client/httpapi/{abyfy_errors.go => errors.go} (88%) rename client/httpapi/{abyfy_errors_test.go => errors_test.go} (84%) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index e78ba2281..0ee838e83 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, abyfyIpldNotFoundFallbackToError(resp.Error) + return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) + return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, abyfyIpldNotFoundFallbackToError(err) + return nil, parseIPLDNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/errors.go similarity index 88% rename from client/httpapi/abyfy_errors.go rename to client/httpapi/errors.go index e4f4afb7c..a527e5452 100644 --- a/client/httpapi/abyfy_errors.go +++ b/client/httpapi/errors.go @@ -27,8 +27,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func abyfyIpldNotFoundFallbackToMSG(msg string) error { - err, handled := abyfyIpldNotFound(msg) +func parseIPLDNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseIPLDNotFound(msg) if handled { return err } @@ -36,8 +36,8 @@ func abyfyIpldNotFoundFallbackToMSG(msg string) error { return errors.New(msg) } -func abyfyIpldNotFoundFallbackToError(msg error) error { - err, handled := abyfyIpldNotFound(msg.Error()) +func parseIPLDNotFoundWithFallbackToError(msg error) error { + err, handled := parseIPLDNotFound(msg.Error()) if handled { return err } @@ -47,7 +47,7 @@ func abyfyIpldNotFoundFallbackToError(msg error) error { // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func abyfyIpldNotFound(msg string) (error, bool) { +func parseIPLDNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/errors_test.go similarity index 84% rename from client/httpapi/abyfy_errors_test.go rename to client/httpapi/errors_test.go index fe1cf0707..08def204d 100644 --- a/client/httpapi/abyfy_errors_test.go +++ b/client/httpapi/errors_test.go @@ -12,10 +12,10 @@ import ( var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} -func doAbyfyIpldNotFoundTest(t *testing.T, original error) { +func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -30,8 +30,8 @@ func doAbyfyIpldNotFoundTest(t *testing.T, original error) { } } -func TestAbyfyIpldNotFound(t *testing.T) { - if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { +func TestParseIPLDNotFound(t *testing.T) { + if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -54,7 +54,7 @@ func TestAbyfyIpldNotFound(t *testing.T) { err = fmt.Errorf(wrap, err) } - doAbyfyIpldNotFoundTest(t, err) + doParseIpldNotFoundTest(t, err) } } } From a2a60768ea7e5d5dad1ad1f0dcf8811cca24d3eb Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:36:11 +0200 Subject: [PATCH 099/112] fix: parseIPLDErrNotFound test This commit was moved from ipfs/go-ipfs-http-client@2e09c4b3ab580f71c652c82926ba7b1580a2b495 --- client/httpapi/errors_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 08def204d..502c10966 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -24,9 +24,9 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } originalNotFound := ipld.IsNotFound(original) - rebuiltNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(rebuilt) if originalNotFound != rebuiltNotFound { - t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound) } } From ddd36645b25c97651deb81c092c7669efabd959e Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:43:27 +0200 Subject: [PATCH 100/112] feat: update the error parsing for go-ipld-format to v0.4.0 This commit was moved from ipfs/go-ipfs-http-client@296534fd1647a1ec9cae08314daf41ec96d55763 --- client/httpapi/errors.go | 54 +++++++++++++++++++++-------------- client/httpapi/errors_test.go | 14 +++++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index a527e5452..f42d92575 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -45,6 +45,18 @@ func parseIPLDNotFoundWithFallbackToError(msg error) error { return msg } +// Use a string to move it into RODATA +// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) +const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +func notAsciiLetterOrDigits(r rune) bool { + if r > 'z' { + return true + } + + return notAsciiLetterOrDigitsLUT[r] > 0 +} + // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseIPLDNotFound(msg string) (error, bool) { @@ -53,46 +65,46 @@ func parseIPLDNotFound(msg string) (error, bool) { } // The patern we search for is: - // node not found (fallback) - // or - // CID not found (here we parse the CID) - notFoundIndex := strings.LastIndex(msg, " not found") + const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ + // We try to parse the CID, if it's invalid we give up and return a simple text error. + // We also accept "node" in place of the CID because that means it's an Undefined CID. - if notFoundIndex == -1 { - // Unknown, ot found not found + keyIndex := strings.Index(msg, ipldErrNotFoundKey) + + if keyIndex < 0 { // Unknown error return nil, false } - preNotFound := msg[:notFoundIndex] + cidStart := keyIndex + len(ipldErrNotFoundKey) + msgPostKey := msg[cidStart:] var c cid.Cid - var preIndex int - if strings.HasSuffix(preNotFound, "node") { + var postIndex int + if strings.HasPrefix(msgPostKey, "node") { // Fallback case c = cid.Undef - preIndex = notFoundIndex - len("node") + postIndex = len("node") } else { - // Assume that CIDs does not include whitespace to pull out the CID - preIndex = strings.LastIndexByte(preNotFound, ' ') - // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space - preIndex++ + // Assume that CIDs only contain a-zA-Z0-9 characters. + // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. + postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + if postIndex < 0 { + postIndex = len(msgPostKey) + } + var err error - c, err = cid.Decode(preNotFound[preIndex:]) + c, err = cid.Decode(msgPostKey[:postIndex]) if err != nil { // Unknown return nil, false } } - postIndex := notFoundIndex + len(" not found") - err := ipld.ErrNotFound{Cid: c} - - pre := msg[:preIndex] - post := msg[postIndex:] + pre := msg[:keyIndex] + post := msgPostKey[postIndex:] if len(pre) > 0 || len(post) > 0 { - // We have some text to wrap arround the ErrNotFound one return prePostWrappedNotFoundError{ pre: pre, post: post, diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 502c10966..1b7de8798 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,6 +3,7 @@ package httpapi import ( "errors" "fmt" + "strings" "testing" "github.com/ipfs/go-cid" @@ -42,9 +43,8 @@ func TestParseIPLDNotFound(t *testing.T) { "%w is wrong", } { for _, err := range [...]error{ - errors.New("file not found"), - errors.New(" not found"), - errors.New("Bad_CID not found"), + errors.New("ipld: could not find "), + errors.New("ipld: could not find Bad_CID"), errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, @@ -58,3 +58,11 @@ func TestParseIPLDNotFound(t *testing.T) { } } } + +func TestNotAsciiLetterOrDigits(t *testing.T) { + for i := rune(0); i <= 256; i++ { + if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { + t.Errorf("%q is incorrectly identified", i) + } + } +} From 775bcb7f09de0d13a64b013ccaefa253e44c3403 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 1 Apr 2022 00:23:55 +0200 Subject: [PATCH 101/112] feat: add blockstore: block not found matching too This commit was moved from ipfs/go-ipfs-http-client@a3354f062c97bcb6412e74ecfb24e84208edc479 --- client/httpapi/block.go | 6 ++-- client/httpapi/errors.go | 56 +++++++++++++++++++++++++++++++---- client/httpapi/errors_test.go | 25 ++++++++++++++-- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 0ee838e83..c074f7940 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) + return nil, parseErrNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) + return parseErrNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, parseIPLDNotFoundWithFallbackToError(err) + return nil, parseErrNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f42d92575..b2a6f86de 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -8,6 +8,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +// This file handle parsing and returning the correct ABI based errors from error messages + type prePostWrappedNotFoundError struct { pre string post string @@ -27,8 +29,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func parseIPLDNotFoundWithFallbackToMSG(msg string) error { - err, handled := parseIPLDNotFound(msg) +func parseErrNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseErrNotFound(msg) if handled { return err } @@ -36,8 +38,8 @@ func parseIPLDNotFoundWithFallbackToMSG(msg string) error { return errors.New(msg) } -func parseIPLDNotFoundWithFallbackToError(msg error) error { - err, handled := parseIPLDNotFound(msg.Error()) +func parseErrNotFoundWithFallbackToError(msg error) error { + err, handled := parseErrNotFound(msg.Error()) if handled { return err } @@ -57,13 +59,25 @@ func notAsciiLetterOrDigits(r rune) bool { return notAsciiLetterOrDigitsLUT[r] > 0 } -// This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func parseIPLDNotFound(msg string) (error, bool) { +func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } + if err, handled := parseIPLDErrNotFound(msg); handled { + return err, true + } + + if err, handled := parseBlockstoreNotFound(msg); handled { + return err, true + } + + return nil, false +} + +//lint:ignore ST1008 using error as values +func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ // We try to parse the CID, if it's invalid we give up and return a simple text error. @@ -114,3 +128,33 @@ func parseIPLDNotFound(msg string) (error, bool) { return err, true } + +// This is a simple error type that just return msg as Error(). +// But that also match ipld.ErrNotFound when called with Is(err). +// That is needed to keep compatiblity with code that use string.Contains(err.Error(), "blockstore: block not found") +// and code using ipld.ErrNotFound +type blockstoreNotFoundMatchingIPLDErrNotFound struct { + msg string +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string { + return e.Error() +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string { + return e.msg +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { + _, ok := err.(ipld.ErrNotFound) + return ok +} + +//lint:ignore ST1008 using error as values +func parseBlockstoreNotFound(msg string) (error, bool) { + if !strings.Contains(msg, "blockstore: block not found") { + return nil, false + } + + return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true +} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 1b7de8798..09437bfbf 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -16,7 +16,7 @@ var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1 func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) + rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -32,7 +32,7 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } func TestParseIPLDNotFound(t *testing.T) { - if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { + if err := parseErrNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -59,6 +59,27 @@ func TestParseIPLDNotFound(t *testing.T) { } } +func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { + if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) { + t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false") + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doParseIpldNotFoundTest(t, err) + } +} + func TestNotAsciiLetterOrDigits(t *testing.T) { for i := rune(0); i <= 256; i++ { if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { From 54f6e90870c61a686f57b0bd0f1ad4923615897b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 03:25:39 +0200 Subject: [PATCH 102/112] fix: change CID breaking logic when parsing ipld.ErrNotFound This commit was moved from ipfs/go-ipfs-http-client@a3b49352bfd1b885567018092a06f303de18b7aa --- client/httpapi/errors.go | 24 +++++++++--------------- client/httpapi/errors_test.go | 18 +++++++----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index b2a6f86de..f1a57a049 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -47,18 +47,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -// Use a string to move it into RODATA -// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) -const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - -func notAsciiLetterOrDigits(r rune) bool { - if r > 'z' { - return true - } - - return notAsciiLetterOrDigitsLUT[r] > 0 -} - //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { @@ -76,6 +64,12 @@ func parseErrNotFound(msg string) (error, bool) { return nil, false } +// Assume CIDs break on: +// - Whitespaces: " \t\n\r\v\f" +// - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr +// - Double Quotes: "\"" this is for parsing %q and %#v formating +const cidBreakSet = " \t\n\r\v\f;\"" + //lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: @@ -99,9 +93,9 @@ func parseIPLDErrNotFound(msg string) (error, bool) { c = cid.Undef postIndex = len("node") } else { - // Assume that CIDs only contain a-zA-Z0-9 characters. - // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. - postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool { + return strings.ContainsAny(string(r), cidBreakSet) + }) if postIndex < 0 { postIndex = len(msgPostKey) } diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 09437bfbf..86f91cdf5 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "strings" "testing" "github.com/ipfs/go-cid" @@ -36,12 +35,17 @@ func TestParseIPLDNotFound(t *testing.T) { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } - for _, wrap := range [...]string{ + cidBreaks := make([]string, len(cidBreakSet)) + for i, v := range cidBreakSet { + cidBreaks[i] = "%w" + string(v) + } + + for _, wrap := range append(cidBreaks, "", "merkledag: %w", "testing: %w the test", "%w is wrong", - } { + ) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), @@ -79,11 +83,3 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { doParseIpldNotFoundTest(t, err) } } - -func TestNotAsciiLetterOrDigits(t *testing.T) { - for i := rune(0); i <= 256; i++ { - if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { - t.Errorf("%q is incorrectly identified", i) - } - } -} From f2f2109bc1e8f5b9c636597eb7478e0a4ddca427 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 04:24:12 +0200 Subject: [PATCH 103/112] test: add a false case test to blockstore parsing This commit was moved from ipfs/go-ipfs-http-client@75f597aa16c512ec02f549836e998f1fc29a3846 --- client/httpapi/errors_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 86f91cdf5..7709fc9c5 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -74,12 +74,15 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { "testing: %w the test", "%w is wrong", } { - var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + for _, err := range [...]error{ + errors.New("network connection timeout"), + blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } - if wrap != "" { - err = fmt.Errorf(wrap, err) + doParseIpldNotFoundTest(t, err) } - - doParseIpldNotFoundTest(t, err) } } From 60a548b621ff229a648a268c4d06f6e206ad6099 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 5 Apr 2022 20:11:06 +0200 Subject: [PATCH 104/112] correctness: only match CIDs matching go-cid.Cid.String output This commit was moved from ipfs/go-ipfs-http-client@34cc489461985cf4139819e589fdd09bdc034c24 --- client/httpapi/errors.go | 19 +++++++++++++++++-- client/httpapi/errors_test.go | 7 +++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f1a57a049..1ccf6182c 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -3,9 +3,11 @@ package httpapi import ( "errors" "strings" + "unicode/utf8" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" ) // This file handle parsing and returning the correct ABI based errors from error messages @@ -97,15 +99,28 @@ func parseIPLDErrNotFound(msg string) (error, bool) { return strings.ContainsAny(string(r), cidBreakSet) }) if postIndex < 0 { + // no breakage meaning the string look like this something + "ipld: could not find bafy" postIndex = len(msgPostKey) } + cidStr := msgPostKey[:postIndex] + var err error - c, err = cid.Decode(msgPostKey[:postIndex]) + c, err = cid.Decode(cidStr) if err != nil { - // Unknown + // failed to decode CID give up return nil, false } + + // check that the CID is either a CIDv0 or a base32 multibase + // because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently + if c.Version() != 0 { + baseRune, _ := utf8.DecodeRuneInString(cidStr) + if baseRune == utf8.RuneError || baseRune != mbase.Base32 { + // not a multibase we expect, give up + return nil, false + } + } } err := ipld.ErrNotFound{Cid: c} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 7709fc9c5..c8b98d08e 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" ) @@ -40,6 +41,11 @@ func TestParseIPLDNotFound(t *testing.T) { cidBreaks[i] = "%w" + string(v) } + base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC) + if err != nil { + t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error()) + } + for _, wrap := range append(cidBreaks, "", "merkledag: %w", @@ -49,6 +55,7 @@ func TestParseIPLDNotFound(t *testing.T) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), + errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, From fd209019f27c7a16f94bfc41ebc7b815daee9903 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 21 Apr 2022 22:18:19 +0200 Subject: [PATCH 105/112] fix: interop with 'block put' from go-ipfs 0.13 (#158) * chore: interop with go-ipfs 0.13 Applies necessary changes to ensure 'block/put' works and is backward-compatible. Context: https://github.com/ipfs/go-ipfs/pull/8568 * chore: 0.3.1 bumping as patch because we bumped to 0.3.0 recently, as part of other (unreleased) go-ipfs 0.13 work This commit was moved from ipfs/go-ipfs-http-client@ecf364c9898d78034e3232574aae70d1bf85357c --- client/httpapi/block.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index c074f7940..7b4cf0eda 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -7,9 +7,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) @@ -31,20 +32,33 @@ func (s *blockStat) Path() path.Resolved { } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { - options, _, err := caopts.BlockPutOptions(opts...) + options, err := caopts.BlockPutOptions(opts...) + px := options.CidPrefix if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] + mht, ok := mh.Codes[px.MhType] if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + return nil, fmt.Errorf("unknowm mhType %d", px.MhType) + } + + var cidOptKey, cidOptVal string + switch { + case px.Version == 0 && px.Codec == cid.DagProtobuf: + // ensure legacy --format=v0 passes as BlockPutOption still works + cidOptKey = "format" + cidOptVal = "v0" + default: + // pass codec as string + cidOptKey = "cid-codec" + cidOptVal = mc.Code(px.Codec).String() } req := api.core().Request("block/put"). Option("mhtype", mht). - Option("mhlen", options.MhLength). - Option("format", options.Codec). + Option("mhlen", px.MhLength). + Option(cidOptKey, cidOptVal). Option("pin", options.Pin). FileBody(r) From 1cdb9adf898a8a2b9d2cd0d3fe8ba6467a822166 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 23 Jun 2022 22:39:01 +0200 Subject: [PATCH 106/112] fix: interop with go-ipfs 0.13 (#160) This ensures cid-codec introduced in https://github.com/ipfs/go-ipfs/pull/8568 gets correctly passed (+ we maintain backward-compatibility with CIDv0) This commit was moved from ipfs/go-ipfs-http-client@9c9f43fd9ca36c7ebdc664bb6936a6fe9881255b --- client/httpapi/dag.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 15d9a9c62..879c1e499 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,12 @@ import ( "fmt" "io/ioutil" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + multicodec "github.com/multiformats/go-multicodec" ) type httpNodeAdder HttpApi @@ -56,13 +57,21 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() - format := cid.CodecToStr[prefix.Codec] + + // preserve 'cid-codec' when sent over HTTP + cidCodec := multicodec.Code(prefix.Codec).String() + + // 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80 + // but we still support it here for backward-compatibility with use of CIDv0 + format := "" if prefix.Version == 0 { + cidCodec = "" format = "v0" } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { From 0fff1d5d8e4cb2debaff1ee4e85b1266907101af Mon Sep 17 00:00:00 2001 From: web3-bot Date: Fri, 23 Sep 2022 07:43:33 +0000 Subject: [PATCH 107/112] stop using the deprecated io/ioutil package This commit was moved from ipfs/go-ipfs-http-client@026ba730a1fe9a4a25a22449bec8d69c2262ccfc --- client/httpapi/api.go | 3 +-- client/httpapi/api_test.go | 3 +-- client/httpapi/apifile.go | 3 +-- client/httpapi/dag.go | 4 ++-- client/httpapi/name.go | 4 ++-- client/httpapi/object.go | 5 ++--- client/httpapi/requestbuilder.go | 5 ++--- client/httpapi/response.go | 9 ++++----- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f589e4267..97440a724 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -74,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { apiFile := filepath.Join(baseDir, DefaultApiFile) - api, err := ioutil.ReadFile(apiFile) + api, err := os.ReadFile(apiFile) if err != nil { return nil, err } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index d5f27f0ec..5960ea2c0 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,7 +2,6 @@ package httpapi import ( "context" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -92,7 +91,7 @@ func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n i func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { - dir, err := ioutil.TempDir("", "httpapi-tb-") + dir, err := os.MkdirTemp("", "httpapi-tb-") if err != nil { return nil, err } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec2f7588d..c4884b924 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" @@ -113,7 +112,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) + r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 879c1e499..f32c67c42 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -24,7 +24,7 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) return nil, err } - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 47227a2ab..b6a783603 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/options/namesys" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6ec7f5503..894369223 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -5,13 +5,12 @@ import ( "context" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -71,7 +70,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { if err != nil { return nil, err } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 7012a8935..039bca036 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,11 +5,10 @@ import ( "context" "fmt" "io" - "io/ioutil" "strconv" "strings" - "github.com/ipfs/go-ipfs-files" + files "github.com/ipfs/go-ipfs-files" ) type RequestBuilder interface { @@ -59,7 +58,7 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder { // FileBody sets the request body to the given reader wrapped into multipartreader. func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { - pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 95cbf13ec..8a491ab73 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -45,7 +44,7 @@ func (r *Response) Close() error { if r.Output != nil { // drain output (response body) - _, err1 := io.Copy(ioutil.Discard, r.Output) + _, err1 := io.Copy(io.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -117,7 +116,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" case contentType == "text/plain": - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } @@ -140,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // This is a server-side bug (probably). e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) } @@ -150,7 +149,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } From 1193b2978f7859ed8e1df4be50e980ff1bde7015 Mon Sep 17 00:00:00 2001 From: galargh Date: Sat, 1 Oct 2022 17:37:23 +0200 Subject: [PATCH 108/112] chore: remove unused linter directives This commit was moved from ipfs/go-ipfs-http-client@c4fc1a7740bfaad2abcba2563d569422e525324c --- client/httpapi/errors.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index 1ccf6182c..59e4ad705 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -49,7 +49,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path @@ -72,7 +71,6 @@ func parseErrNotFound(msg string) (error, bool) { // - Double Quotes: "\"" this is for parsing %q and %#v formating const cidBreakSet = " \t\n\r\v\f;\"" -//lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ @@ -159,7 +157,6 @@ func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { return ok } -//lint:ignore ST1008 using error as values func parseBlockstoreNotFound(msg string) (error, bool) { if !strings.Contains(msg, "blockstore: block not found") { return nil, false From 75b6f45f765237d611168a5e1c09915f380e7bd7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 6 Feb 2023 15:32:57 +0100 Subject: [PATCH 109/112] chore: bumps for Kubo 0.18 and Routing API with stub for Put This commit was moved from ipfs/go-ipfs-http-client@c076c3cb71d08fb9448ae9bf6916dab137322722 --- client/httpapi/api.go | 4 +++ client/httpapi/apifile.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/dht.go | 4 +-- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/routing.go | 55 ++++++++++++++++++++++++++++++++ client/httpapi/swarm.go | 6 ++-- client/httpapi/unixfs.go | 2 +- 11 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/httpapi/routing.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 97440a724..10df94603 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -195,3 +195,7 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { func (api *HttpApi) PubSub() iface.PubSubAPI { return (*PubsubAPI)(api) } + +func (api *HttpApi) Routing() iface.RoutingAPI { + return (*RoutingAPI)(api) +} diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index c4884b924..80fd13cd4 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -7,7 +7,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index f32c67c42..2fbfea1ff 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 0a78514f9..ebecfcaeb 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,8 +6,8 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" ) type DhtAPI HttpApi diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 78b67517c..b785bbf45 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 72f592376..ec45d1a51 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 039bca036..bc6f4c8a9 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 8a491ab73..99932ca0d 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -12,7 +12,7 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/routing.go b/client/httpapi/routing.go new file mode 100644 index 000000000..18f533276 --- /dev/null +++ b/client/httpapi/routing.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + + "github.com/libp2p/go-libp2p/core/routing" +) + +type RoutingAPI HttpApi + +func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { + resp, err := api.core().Request("routing/get", key).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Close() + + var out routing.QueryEvent + + dec := json.NewDecoder(resp.Output) + if err := dec.Decode(&out); err != nil { + return nil, err + } + + res, err := base64.StdEncoding.DecodeString(out.Extra) + if err != nil { + return nil, err + } + + return res, nil +} + +func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte) error { + resp, err := api.core().Request("routing/put", key). + FileBody(bytes.NewReader(value)). + Send(ctx) + + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + return nil +} + +func (api *RoutingAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 1cb0d91df..c5dfbe564 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,9 +5,9 @@ import ( "time" iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5e27fb036..38f23d5b9 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,7 +8,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" From 5c21cf0ce95d7c2710d5bb5dd58e2871c0fbc7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= <92101826+PedrobyJoao@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:38:24 -0300 Subject: [PATCH 110/112] docs: adding example of connection and pinning to README (#173) Co-authored-by: JPexplorer This commit was moved from ipfs/go-ipfs-http-client@b0de2b028e6ef03d4dba3ed105641ad299d4f7be --- client/httpapi/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index d3e14ac3f..da25f24fe 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -17,6 +17,41 @@ greatest features, please use _this_ package. https://godoc.org/github.com/ipfs/go-ipfs-http-api +### Example + +Pin file on your local IPFS node based on its CID: + +```go +package main + +import ( + "context" + "fmt" + + ipfsClient "github.com/ipfs/go-ipfs-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" +) + +func main() { + // "Connect" to local node + node, err := ipfsClient.NewLocalApi() + if err != nil { + fmt.Printf(err) + return + } + // Pin a given file by its CID + ctx := context.Background() + cid := "bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku" + p := path.New(cid) + err = node.Pin().Add(ctx, p) + if err != nil { + fmt.Printf(err) + return + } + return +} +``` + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! From 8bebfc00b7192de63b79082bd2666d8d1bf76126 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 16 Apr 2023 12:31:09 +0200 Subject: [PATCH 111/112] chore: migrate from go-libipfs to boxo This commit was moved from ipfs/go-ipfs-http-client@ae996cbe5a91708d245100267c45094ab52c42ad --- client/httpapi/api.go | 4 ++-- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 6 +++--- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 4 ++-- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 8 ++++---- client/httpapi/object.go | 10 +++++----- client/httpapi/path.go | 4 ++-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 4 ++-- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/swarm.go | 2 +- client/httpapi/unixfs.go | 12 ++++++------ 16 files changed, 44 insertions(+), 45 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 10df94603..b7cfb2b52 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5960ea2c0..2832d722d 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -12,10 +12,9 @@ import ( "testing" "time" - iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/path" - - "github.com/ipfs/interface-go-ipfs-core/tests" + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/coreiface/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" testbedi "github.com/ipfs/iptb/testbed/interfaces" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 80fd13cd4..25fd7c3b3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 7b4cf0eda..2a794c26f 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 2fbfea1ff..795e1d78d 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,11 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" ) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index ebecfcaeb..a2910fef6 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index b785bbf45..434e98fe5 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,9 @@ import ( "context" "errors" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b6a783603..f82f69f3a 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/coreiface/path" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 894369223..4e3b9ef6b 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,13 +6,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ft "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 619dd0e0d..d69d425ab 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,10 +3,10 @@ package httpapi import ( "context" + "github.com/ipfs/boxo/coreiface/path" + ipfspath "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core/path" ) func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 13de2d389..30a3d7b7a 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -5,10 +5,10 @@ import ( "encoding/json" "strings" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index ec45d1a51..28f1ef8e6 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,8 +6,8 @@ import ( "encoding/json" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index bc6f4c8a9..476aed786 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/boxo/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 99932ca0d..189b43671 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -10,9 +10,9 @@ import ( "net/url" "os" + "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index c5dfbe564..9b073078d 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - iface "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/boxo/coreiface" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 38f23d5b9..b9c34c59f 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -7,13 +7,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - unixfs_pb "github.com/ipfs/go-unixfs/pb" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) From 7d8262307cb40c1f21fd79fe52cdfd36a7dcb5cd Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 15 May 2023 17:18:51 +0100 Subject: [PATCH 112/112] fix: use https URI when multiaddress specifies tls (#177) Currently any clients created through `NewApiWithClient` will make a HTTP request to the api, even if the multiaddress specifies TLS or (the deprecated multiaddr option) https. This commit addresses this by having NewApiWithClient iterate the available protocols for the multiaddress, specifying the URL proto as https if it finds TLS or HTTPS is specified. The default continues to be http for those multiaddresses that do not specify these options. Should resolve #176 This commit was moved from ipfs/go-ipfs-http-client@7e1de1f7ccfa8cd1f616e94664886fe04d70806e --- client/httpapi/api.go | 16 +++++++++++++++- client/httpapi/api_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b7cfb2b52..5584de85d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -107,7 +107,21 @@ func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { } } - return NewURLApiWithClient(url, c) + proto := "http://" + + // By default, DialArgs is going to provide details suitable for connecting + // a socket to, but not really suitable for making an informed choice of http + // protocol. For multiaddresses specifying tls and/or https we want to make + // a https request instead of a http request. + protocols := a.Protocols() + for _, p := range protocols { + if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS { + proto = "https://" + break + } + } + + return NewURLApiWithClient(proto+url, c) } func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2832d722d..a6d684b0d 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -246,3 +246,31 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } } + +func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) { + testcases := []struct { + address string + expected string + }{ + {address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"}, + {address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"}, + } + + for _, tc := range testcases { + address, err := ma.NewMultiaddr(tc.address) + if err != nil { + t.Fatal(err) + } + + api, err := NewApiWithClient(address, &http.Client{}) + if err != nil { + t.Fatal(err) + } + + if api.url != tc.expected { + t.Errorf("Expected = %s; got %s", tc.expected, api.url) + } + } +}