From 6d85aff407d3fb3eb6d2549fbac94615fd147b49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 6 Nov 2018 13:13:15 +0100
Subject: [PATCH 01/64] Initial structure, path stuff

This commit was moved from ipfs/go-ipfs-http-client@93943f7f5671948b0d2aee19c30cf689307faf13
---
 client/httpapi/api.go            | 127 +++++++++++++++++++++++++++++
 client/httpapi/name.go           |  35 ++++++++
 client/httpapi/path.go           |  48 +++++++++++
 client/httpapi/request.go        |  34 ++++++++
 client/httpapi/requestbuilder.go | 100 +++++++++++++++++++++++
 client/httpapi/response.go       | 132 +++++++++++++++++++++++++++++++
 6 files changed, 476 insertions(+)
 create mode 100644 client/httpapi/api.go
 create mode 100644 client/httpapi/name.go
 create mode 100644 client/httpapi/path.go
 create mode 100644 client/httpapi/request.go
 create mode 100644 client/httpapi/requestbuilder.go
 create mode 100644 client/httpapi/response.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
new file mode 100644
index 000000000..82e030350
--- /dev/null
+++ b/client/httpapi/api.go
@@ -0,0 +1,127 @@
+package httpapi
+
+import (
+	"github.com/pkg/errors"
+	"io/ioutil"
+	gohttp "net/http"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	homedir "github.com/mitchellh/go-homedir"
+	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr-net"
+	)
+
+const (
+	DefaultPathName = ".ipfs"
+	DefaultPathRoot = "~/" + DefaultPathName
+	DefaultApiFile  = "api"
+	EnvDir          = "IPFS_PATH"
+)
+
+var ErrNotImplemented = errors.New("not implemented")
+
+type HttpApi struct {
+	url     string
+	httpcli *gohttp.Client
+}
+
+func NewLocalApi() iface.CoreAPI {
+	baseDir := os.Getenv(EnvDir)
+	if baseDir == "" {
+		baseDir = DefaultPathRoot
+	}
+
+	baseDir, err := homedir.Expand(baseDir)
+	if err != nil {
+		return nil
+	}
+
+	apiFile := path.Join(baseDir, DefaultApiFile)
+
+	if _, err := os.Stat(apiFile); err != nil {
+		return nil
+	}
+
+	api, err := ioutil.ReadFile(apiFile)
+	if err != nil {
+		return nil
+	}
+
+	return NewApi(strings.TrimSpace(string(api)))
+}
+
+func NewApi(url string) *HttpApi {
+	c := &gohttp.Client{
+		Transport: &gohttp.Transport{
+			Proxy:             gohttp.ProxyFromEnvironment,
+			DisableKeepAlives: true,
+		},
+	}
+
+	return NewApiWithClient(url, c)
+}
+
+func NewApiWithClient(url string, c *gohttp.Client) *HttpApi {
+	if a, err := ma.NewMultiaddr(url); err == nil {
+		_, host, err := manet.DialArgs(a)
+		if err == nil {
+			url = host
+		}
+	}
+
+	return &HttpApi{
+		url:     url,
+		httpcli: c,
+	}
+}
+
+func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
+	return &RequestBuilder{
+		command: command,
+		args:    args,
+		shell:   api,
+	}
+}
+
+func (api *HttpApi) Unixfs() iface.UnixfsAPI {
+	return nil
+}
+
+func (api *HttpApi) Block() iface.BlockAPI {
+	return nil
+}
+
+func (api *HttpApi) Dag() iface.DagAPI {
+	return nil
+}
+
+func (api *HttpApi) Name() iface.NameAPI {
+	return (*NameAPI)(api)
+}
+
+func (api *HttpApi) Key() iface.KeyAPI {
+	return nil
+}
+
+func (api *HttpApi) Pin() iface.PinAPI {
+	return nil
+}
+
+func (api *HttpApi) Object() iface.ObjectAPI {
+	return nil
+}
+
+func (api *HttpApi) Dht() iface.DhtAPI {
+	return nil
+}
+
+func (api *HttpApi) Swarm() iface.SwarmAPI {
+	return nil
+}
+
+func (api *HttpApi) PubSub() iface.PubSubAPI {
+	return nil
+}
diff --git a/client/httpapi/name.go b/client/httpapi/name.go
new file mode 100644
index 000000000..41426ef57
--- /dev/null
+++ b/client/httpapi/name.go
@@ -0,0 +1,35 @@
+package httpapi
+
+import (
+	"context"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+)
+
+type NameAPI HttpApi
+
+func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) {
+	return nil, ErrNotImplemented
+}
+
+func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) {
+	return nil, ErrNotImplemented
+}
+
+func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) {
+	// TODO: options!
+
+	req := api.core().request("name/resolve")
+	req.Arguments(name)
+
+	var out struct{ Path string }
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	return iface.ParsePath(out.Path)
+}
+
+func (api *NameAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}
diff --git a/client/httpapi/path.go b/client/httpapi/path.go
new file mode 100644
index 000000000..28656fbd4
--- /dev/null
+++ b/client/httpapi/path.go
@@ -0,0 +1,48 @@
+package httpapi
+
+import (
+	"context"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+
+	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
+	ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path"
+	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
+)
+
+func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) {
+	var out struct {
+		Cid     cid.Cid
+		RemPath string
+	}
+
+	//TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help
+
+	var err error
+	if path.Namespace() == "ipns" {
+		if path, err = api.Name().Resolve(ctx, path.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	// TODO:
+	ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1])
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil
+}
+
+func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) {
+	return nil, ErrNotImplemented
+}
diff --git a/client/httpapi/request.go b/client/httpapi/request.go
new file mode 100644
index 000000000..58c61ac67
--- /dev/null
+++ b/client/httpapi/request.go
@@ -0,0 +1,34 @@
+package httpapi
+
+import (
+	"context"
+	"io"
+	"strings"
+)
+
+type Request struct {
+	ApiBase string
+	Command string
+	Args    []string
+	Opts    map[string]string
+	Body    io.Reader
+	Headers map[string]string
+}
+
+func NewRequest(ctx context.Context, url, command string, args ...string) *Request {
+	if !strings.HasPrefix(url, "http") {
+		url = "http://" + url
+	}
+
+	opts := map[string]string{
+		"encoding":        "json",
+		"stream-channels": "true",
+	}
+	return &Request{
+		ApiBase: url + "/api/v0",
+		Command: command,
+		Args:    args,
+		Opts:    opts,
+		Headers: make(map[string]string),
+	}
+}
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
new file mode 100644
index 000000000..9ccc8cf97
--- /dev/null
+++ b/client/httpapi/requestbuilder.go
@@ -0,0 +1,100 @@
+package httpapi
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// RequestBuilder is an IPFS commands request builder.
+type RequestBuilder struct {
+	command string
+	args    []string
+	opts    map[string]string
+	headers map[string]string
+	body    io.Reader
+
+	shell *HttpApi
+}
+
+// Arguments adds the arguments to the args.
+func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder {
+	r.args = append(r.args, args...)
+	return r
+}
+
+// BodyString sets the request body to the given string.
+func (r *RequestBuilder) BodyString(body string) *RequestBuilder {
+	return r.Body(strings.NewReader(body))
+}
+
+// BodyBytes sets the request body to the given buffer.
+func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder {
+	return r.Body(bytes.NewReader(body))
+}
+
+// Body sets the request body to the given reader.
+func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder {
+	r.body = body
+	return r
+}
+
+// Option sets the given option.
+func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder {
+	var s string
+	switch v := value.(type) {
+	case bool:
+		s = strconv.FormatBool(v)
+	case string:
+		s = v
+	case []byte:
+		s = string(v)
+	default:
+		// slow case.
+		s = fmt.Sprint(value)
+	}
+	if r.opts == nil {
+		r.opts = make(map[string]string, 1)
+	}
+	r.opts[key] = s
+	return r
+}
+
+// Header sets the given header.
+func (r *RequestBuilder) Header(name, value string) *RequestBuilder {
+	if r.headers == nil {
+		r.headers = make(map[string]string, 1)
+	}
+	r.headers[name] = value
+	return r
+}
+
+// Send sends the request and return the response.
+func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) {
+	req := NewRequest(ctx, r.shell.url, r.command, r.args...)
+	req.Opts = r.opts
+	req.Headers = r.headers
+	req.Body = r.body
+	return req.Send(r.shell.httpcli)
+}
+
+// Exec sends the request a request and decodes the response.
+func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error {
+	httpRes, err := r.Send(ctx)
+	if err != nil {
+		return err
+	}
+
+	if res == nil {
+		httpRes.Close()
+		if httpRes.Error != nil {
+			return httpRes.Error
+		}
+		return nil
+	}
+
+	return httpRes.Decode(res)
+}
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
new file mode 100644
index 000000000..27709769b
--- /dev/null
+++ b/client/httpapi/response.go
@@ -0,0 +1,132 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	files "github.com/ipfs/go-ipfs-files"
+)
+
+type Response struct {
+	Output io.ReadCloser
+	Error  *Error
+}
+
+func (r *Response) Close() error {
+	if r.Output != nil {
+		// always drain output (response body)
+		ioutil.ReadAll(r.Output)
+		return r.Output.Close()
+	}
+	return nil
+}
+
+func (r *Response) Decode(dec interface{}) error {
+	defer r.Close()
+	if r.Error != nil {
+		return r.Error
+	}
+
+	return json.NewDecoder(r.Output).Decode(dec)
+}
+
+type Error struct {
+	Command string
+	Message string
+	Code    int
+}
+
+func (e *Error) Error() string {
+	var out string
+	if e.Command != "" {
+		out = e.Command + ": "
+	}
+	if e.Code != 0 {
+		out = fmt.Sprintf("%s%d: ", out, e.Code)
+	}
+	return out + e.Message
+}
+
+func (r *Request) Send(c *http.Client) (*Response, error) {
+	url := r.getURL()
+	req, err := http.NewRequest("POST", url, r.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add any headers that were supplied via the RequestBuilder.
+	for k, v := range r.Headers {
+		req.Header.Add(k, v)
+	}
+
+	if fr, ok := r.Body.(*files.MultiFileReader); ok {
+		req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary())
+		req.Header.Set("Content-Disposition", "form-data: name=\"files\"")
+	}
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	contentType := resp.Header.Get("Content-Type")
+	parts := strings.Split(contentType, ";")
+	contentType = parts[0]
+
+	nresp := new(Response)
+
+	nresp.Output = resp.Body
+	if resp.StatusCode >= http.StatusBadRequest {
+		e := &Error{
+			Command: r.Command,
+		}
+		switch {
+		case resp.StatusCode == http.StatusNotFound:
+			e.Message = "command not found"
+		case contentType == "text/plain":
+			out, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err)
+			}
+			e.Message = string(out)
+		case contentType == "application/json":
+			if err = json.NewDecoder(resp.Body).Decode(e); err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err)
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType)
+			out, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err)
+			}
+			e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out)
+		}
+		nresp.Error = e
+		nresp.Output = nil
+
+		// drain body and close
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}
+
+	return nresp, nil
+}
+
+func (r *Request) getURL() string {
+
+	values := make(url.Values)
+	for _, arg := range r.Args {
+		values.Add("arg", arg)
+	}
+	for k, v := range r.Opts {
+		values.Add(k, v)
+	}
+
+	return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode())
+}

From 6c927fd9624807e124f0376445c41daaa975fbd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 6 Nov 2018 14:38:11 +0100
Subject: [PATCH 02/64] Partial ipld node impl

This commit was moved from ipfs/go-ipfs-http-client@df916c7849d6cfd7f479c4fe80c0f75d6705a0fc
---
 client/httpapi/api.go      |   6 +--
 client/httpapi/block.go    |  40 ++++++++++++++
 client/httpapi/ipldnode.go | 107 +++++++++++++++++++++++++++++++++++++
 client/httpapi/path.go     |   2 +-
 4 files changed, 151 insertions(+), 4 deletions(-)
 create mode 100644 client/httpapi/block.go
 create mode 100644 client/httpapi/ipldnode.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 82e030350..cd3fb9fd0 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -1,7 +1,7 @@
 package httpapi
 
 import (
-	"github.com/pkg/errors"
+	"errors"
 	"io/ioutil"
 	gohttp "net/http"
 	"os"
@@ -12,7 +12,7 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
-	)
+)
 
 const (
 	DefaultPathName = ".ipfs"
@@ -91,7 +91,7 @@ func (api *HttpApi) Unixfs() iface.UnixfsAPI {
 }
 
 func (api *HttpApi) Block() iface.BlockAPI {
-	return nil
+	return (*BlockAPI)(api)
 }
 
 func (api *HttpApi) Dag() iface.DagAPI {
diff --git a/client/httpapi/block.go b/client/httpapi/block.go
new file mode 100644
index 000000000..8bdb4c502
--- /dev/null
+++ b/client/httpapi/block.go
@@ -0,0 +1,40 @@
+package httpapi
+
+import (
+	"context"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+)
+
+type BlockAPI HttpApi
+
+func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) {
+	return nil, ErrNotImplemented
+}
+
+func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) {
+	resp, err := api.core().request("block/get", p.String()).Send(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	//TODO: is close on the reader enough?
+	//defer resp.Close()
+
+	//TODO: make blockApi return ReadCloser
+	return resp.Output, resp.Error
+}
+
+func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error {
+	return ErrNotImplemented
+}
+
+func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) {
+	return nil, ErrNotImplemented
+}
+
+func (api *BlockAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
new file mode 100644
index 000000000..a3dd6204b
--- /dev/null
+++ b/client/httpapi/ipldnode.go
@@ -0,0 +1,107 @@
+package httpapi
+
+import (
+	"context"
+	"fmt"
+	"github.com/pkg/errors"
+	"gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
+	"io/ioutil"
+	"strconv"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+
+	ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path"
+	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
+)
+
+type ipldNode struct {
+	ctx  context.Context //TODO: should we re-consider adding ctx to ipld interfaces?
+	path iface.ResolvedPath
+	api  *HttpApi
+}
+
+func (n *ipldNode) RawData() []byte {
+	r, err := n.api.Block().Get(n.ctx, n.path)
+	if err != nil {
+		panic(err) // TODO: eww, should we add errors too / better ideas?
+	}
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		panic(err)
+	}
+
+	return b
+}
+
+func (n *ipldNode) Cid() cid.Cid {
+	return n.path.Cid()
+}
+
+func (n *ipldNode) String() string {
+	return fmt.Sprintf("[Block %s]", n.Cid())
+}
+
+func (n *ipldNode) Loggable() map[string]interface{} {
+	return nil //TODO: we can't really do better here, can we?
+}
+
+// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that.
+// We can also give people a choice
+func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) {
+	p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)})
+
+	var out interface{}
+	n.api.request("dag/get", p).Exec(n.ctx, &out)
+
+	// TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack
+	for len(path) > 0 {
+		switch o := out.(type) {
+		case map[string]interface{}:
+			v, ok := o[path[0]]
+			if !ok {
+				// TODO: ipld links
+				return nil, nil, errors.New("no element under this path")
+			}
+			out = v
+		case []interface{}:
+			n, err := strconv.ParseUint(path[0], 10, 32)
+			if err != nil {
+				return nil, nil, err
+			}
+			if len(o) < int(n) {
+				return nil, nil, errors.New("no element under this path")
+			}
+			out = o[n]
+		}
+		path = path[1:]
+	}
+
+	return out, path, nil
+}
+
+func (n *ipldNode) Tree(path string, depth int) []string {
+	panic("implement me")
+}
+
+func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) {
+	panic("implement me")
+}
+
+func (n *ipldNode) Copy() ipld.Node {
+	panic("implement me")
+}
+
+func (n *ipldNode) Links() []*ipld.Link {
+	panic("implement me")
+}
+
+func (n *ipldNode) Stat() (*ipld.NodeStat, error) {
+	panic("implement me")
+}
+
+func (n *ipldNode) Size() (uint64, error) {
+	panic("implement me")
+}
+
+var _ ipld.Node = &ipldNode{}
diff --git a/client/httpapi/path.go b/client/httpapi/path.go
index 28656fbd4..6b6e4b027 100644
--- a/client/httpapi/path.go
+++ b/client/httpapi/path.go
@@ -30,7 +30,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res
 	}
 
 	// TODO:
-	ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath)
+	ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath)
 	if err != nil {
 		return nil, err
 	}

From 1cd2ec05b7524e921f58750a4b9c8f48ad1a2e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 20 Dec 2018 16:04:19 +0100
Subject: [PATCH 03/64] Init gx

This commit was moved from ipfs/go-ipfs-http-client@e06cddbedd360b1982e414571dd08ab25df38546
---
 client/httpapi/ipldnode.go | 8 ++++----
 client/httpapi/name.go     | 1 +
 client/httpapi/path.go     | 6 +++---
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
index a3dd6204b..b8e6fba01 100644
--- a/client/httpapi/ipldnode.go
+++ b/client/httpapi/ipldnode.go
@@ -2,16 +2,16 @@ package httpapi
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"github.com/pkg/errors"
-	"gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	"io/ioutil"
 	"strconv"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 
-	ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path"
-	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	ipfspath "github.com/ipfs/go-path"
 )
 
 type ipldNode struct {
diff --git a/client/httpapi/name.go b/client/httpapi/name.go
index 41426ef57..7315ac2c3 100644
--- a/client/httpapi/name.go
+++ b/client/httpapi/name.go
@@ -2,6 +2,7 @@ package httpapi
 
 import (
 	"context"
+
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 )
diff --git a/client/httpapi/path.go b/client/httpapi/path.go
index 6b6e4b027..5701326fc 100644
--- a/client/httpapi/path.go
+++ b/client/httpapi/path.go
@@ -5,9 +5,9 @@ import (
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 
-	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
-	ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path"
-	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
+	cid "github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	ipfspath "github.com/ipfs/go-path"
 )
 
 func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) {

From 16e97bf5e8f80b681c5a8d4b96f406b4b83a8e1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Fri, 21 Dec 2018 18:44:34 +0100
Subject: [PATCH 04/64] It builds

This commit was moved from ipfs/go-ipfs-http-client@a23d794e5fceadb9115d8d0c8466c10e9c5d0891
---
 client/httpapi/api.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index cd3fb9fd0..dc9acb9f6 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
@@ -78,6 +79,10 @@ func NewApiWithClient(url string, c *gohttp.Client) *HttpApi {
 	}
 }
 
+func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) {
+	return nil, ErrNotImplemented
+}
+
 func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 	return &RequestBuilder{
 		command: command,

From fe4c9fd8033c94c5ea707b09e76479a6d018635d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Sun, 30 Dec 2018 04:24:09 +0100
Subject: [PATCH 05/64] Skeleton for tests

This commit was moved from ipfs/go-ipfs-http-client@dfbe0026ad9438d4e795bbb43debeffb6bb9ca1a
---
 client/httpapi/api.go      | 24 ++++++++---
 client/httpapi/api_test.go | 84 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 5 deletions(-)
 create mode 100644 client/httpapi/api_test.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index dc9acb9f6..63ea3aad7 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -35,7 +35,11 @@ func NewLocalApi() iface.CoreAPI {
 		baseDir = DefaultPathRoot
 	}
 
-	baseDir, err := homedir.Expand(baseDir)
+	return NewPathApi(baseDir)
+}
+
+func NewPathApi(p string) iface.CoreAPI {
+	baseDir, err := homedir.Expand(p)
 	if err != nil {
 		return nil
 	}
@@ -51,10 +55,15 @@ func NewLocalApi() iface.CoreAPI {
 		return nil
 	}
 
-	return NewApi(strings.TrimSpace(string(api)))
+	maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api)))
+	if err != nil {
+		return nil
+	}
+
+	return NewApi(maddr)
 }
 
-func NewApi(url string) *HttpApi {
+func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr?
 	c := &gohttp.Client{
 		Transport: &gohttp.Transport{
 			Proxy:             gohttp.ProxyFromEnvironment,
@@ -62,10 +71,15 @@ func NewApi(url string) *HttpApi {
 		},
 	}
 
-	return NewApiWithClient(url, c)
+	return NewApiWithClient(a, c)
 }
 
-func NewApiWithClient(url string, c *gohttp.Client) *HttpApi {
+func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
+	_, url, err := manet.DialArgs(a)
+	if err != nil {
+		return nil // TODO: return that error
+	}
+
 	if a, err := ma.NewMultiaddr(url); err == nil {
 		_, host, err := manet.DialArgs(a)
 		if err == nil {
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
new file mode 100644
index 000000000..b925b5c34
--- /dev/null
+++ b/client/httpapi/api_test.go
@@ -0,0 +1,84 @@
+package httpapi
+
+import (
+	"context"
+	"fmt"
+	"github.com/ipfs/iptb/testbed/interfaces"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/tests"
+
+	local "github.com/ipfs/iptb-plugins/local"
+	"github.com/ipfs/iptb/cli"
+	"github.com/ipfs/iptb/testbed"
+)
+
+type NodeProvider struct{}
+
+func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
+	_, err := testbed.RegisterPlugin(testbed.IptbPlugin{
+		From:        "<builtin>",
+		NewNode:     local.NewNode,
+		GetAttrList: local.GetAttrList,
+		GetAttrDesc: local.GetAttrDesc,
+		PluginName:  local.PluginName,
+		BuiltIn:     true,
+	}, false)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := ioutil.TempDir("", "httpapi-tb-")
+	if err != nil {
+		return nil, err
+	}
+
+	c := cli.NewCli()
+	if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil {
+		return nil, err
+	}
+
+	go func() {
+		<-ctx.Done()
+
+		defer os.Remove(dir)
+
+		defer func() {
+			_ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"})
+		}()
+	}()
+
+	apis := make([]iface.CoreAPI, n)
+
+	for i := range apis {
+		tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default"))
+
+		node, err := tb.Node(i)
+		if err != nil {
+			return nil, err
+		}
+
+		attrNode, ok := node.(testbedi.Attribute)
+		if !ok {
+			return nil, fmt.Errorf("node does not implement attributes")
+		}
+
+		pth, err := attrNode.Attr("path")
+		if err != nil {
+			return nil, err
+		}
+
+		apis[i] = NewPathApi(pth)
+	}
+
+	return apis, nil
+}
+
+func TestHttpApi(t *testing.T) {
+	tests.TestApi(&NodeProvider{})(t)
+}

From e8da6e2cf9a8fd40c5fd960a60ce1b1b27ada713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 7 Jan 2019 23:44:43 +0100
Subject: [PATCH 06/64] Block API

This commit was moved from ipfs/go-ipfs-http-client@d0c98b870e55bed78fa777793e520f08157deb4f
---
 client/httpapi/api.go            |   2 +-
 client/httpapi/api_test.go       |  11 +++-
 client/httpapi/block.go          | 109 ++++++++++++++++++++++++++++---
 client/httpapi/requestbuilder.go |  12 ++++
 4 files changed, 121 insertions(+), 13 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 63ea3aad7..1768e8477 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI {
 }
 
 func (api *HttpApi) Name() iface.NameAPI {
-	return (*NameAPI)(api)
+	return nil
 }
 
 func (api *HttpApi) Key() iface.KeyAPI {
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index b925b5c34..e32c456e5 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -3,7 +3,6 @@ package httpapi
 import (
 	"context"
 	"fmt"
-	"github.com/ipfs/iptb/testbed/interfaces"
 	"io/ioutil"
 	"os"
 	"path"
@@ -16,6 +15,7 @@ import (
 	local "github.com/ipfs/iptb-plugins/local"
 	"github.com/ipfs/iptb/cli"
 	"github.com/ipfs/iptb/testbed"
+	"github.com/ipfs/iptb/testbed/interfaces"
 )
 
 type NodeProvider struct{}
@@ -39,7 +39,14 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 	}
 
 	c := cli.NewCli()
-	if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil {
+
+	initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)}
+	if err := c.Run(initArgs); err != nil {
+		return nil, err
+	}
+
+	startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)}
+	if err := c.Run(startArgs); err != nil {
 		return nil, err
 	}
 
diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index 8bdb4c502..185aa0a42 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -1,17 +1,69 @@
 package httpapi
 
 import (
+	"bytes"
 	"context"
+	"errors"
+	"fmt"
 	"io"
 
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	mh "github.com/multiformats/go-multihash"
 )
 
 type BlockAPI HttpApi
 
-func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) {
-	return nil, ErrNotImplemented
+type blockStat struct {
+	Key  string
+	BSize int `json:"Size"`
+}
+
+func (s *blockStat) Size() int {
+	return s.BSize
+}
+
+func (s *blockStat) valid() (iface.ResolvedPath, error) {
+	c, err := cid.Parse(s.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpldPath(c), nil
+}
+
+func (s *blockStat) Path() iface.ResolvedPath {
+	p, _ := s.valid()
+	return p
+}
+
+func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) {
+	options, _, err := caopts.BlockPutOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	mht, ok := mh.Codes[options.MhType]
+	if !ok {
+		return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+	}
+
+	req := api.core().request("block/put").
+		Option("mhtype", mht).
+		Option("mhlen", options.MhLength).
+		Option("format", options.Codec).
+		FileBody(r)
+
+	var out blockStat
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	if _, err := out.valid(); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
 }
 
 func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) {
@@ -19,20 +71,57 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) {
 	if err != nil {
 		return nil, err
 	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
 
-	//TODO: is close on the reader enough?
-	//defer resp.Close()
+	//TODO: make get return ReadCloser to avoid copying
+	defer resp.Close()
+	b := new(bytes.Buffer)
+	if _, err := io.Copy(b, resp.Output); err != nil {
+		return nil, err
+	}
 
-	//TODO: make blockApi return ReadCloser
-	return resp.Output, resp.Error
+	return b, nil
 }
 
-func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error {
-	return ErrNotImplemented
+func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error {
+	options, err := caopts.BlockRmOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	removedBlock := struct {
+		Hash  string `json:",omitempty"`
+		Error string `json:",omitempty"`
+	}{}
+
+	req := api.core().request("block/rm").
+		Option("force", options.Force).
+		Arguments(p.String())
+
+	if err := req.Exec(ctx, &removedBlock); err != nil {
+		return err
+	}
+
+	if removedBlock.Error != "" {
+		return errors.New(removedBlock.Error)
+	}
+
+	return nil
 }
 
 func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) {
-	return nil, ErrNotImplemented
+	var out blockStat
+	err := api.core().request("block/stat", p.String()).Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := out.valid(); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
 }
 
 func (api *BlockAPI) core() *HttpApi {
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index 9ccc8cf97..6e5a89ebd 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -5,8 +5,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strconv"
 	"strings"
+
+	"github.com/ipfs/go-ipfs-files"
 )
 
 // RequestBuilder is an IPFS commands request builder.
@@ -42,6 +45,15 @@ func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder {
 	return r
 }
 
+// FileBody sets the request body to the given reader wrapped into multipartreader.
+func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder {
+	pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil)
+	d := files.NewMapDirectory(map[string]files.Node{"": pr})
+	r.body = files.NewMultiFileReader(d, false)
+
+	return r
+}
+
 // Option sets the given option.
 func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder {
 	var s string

From ab89e0abf99a0bc42ee2d8fb55a9b3de79573635 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 00:30:22 +0100
Subject: [PATCH 07/64] Partial Unixfs.Add

This commit was moved from ipfs/go-ipfs-http-client@44696b84f59f6f707858787c9a0750f224596635
---
 client/httpapi/api.go    |  4 +-
 client/httpapi/unixfs.go | 95 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100644 client/httpapi/unixfs.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 1768e8477..099b45123 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -106,7 +106,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 }
 
 func (api *HttpApi) Unixfs() iface.UnixfsAPI {
-	return nil
+	return (*UnixfsAPI)(api)
 }
 
 func (api *HttpApi) Block() iface.BlockAPI {
@@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI {
 }
 
 func (api *HttpApi) Name() iface.NameAPI {
-	return nil
+	return (*NameAPI)(api)
 }
 
 func (api *HttpApi) Key() iface.KeyAPI {
diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
new file mode 100644
index 000000000..567f60284
--- /dev/null
+++ b/client/httpapi/unixfs.go
@@ -0,0 +1,95 @@
+package httpapi
+
+import (
+	"context"
+	"fmt"
+	"github.com/ipfs/go-cid"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/ipfs/go-ipfs-files"
+	"github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type addEvent struct {
+	Name  string
+	Hash  string `json:",omitempty"`
+	Bytes int64  `json:",omitempty"`
+	Size  string `json:",omitempty"`
+}
+
+type UnixfsAPI HttpApi
+
+func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) {
+	options, _, err := caopts.UnixfsAddOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	mht, ok := mh.Codes[options.MhType]
+	if !ok {
+		return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+	}
+
+	req := api.core().request("add").
+		Option("hash", mht).
+		Option("chunker", options.Chunker).
+		Option("cid-version", options.CidVersion).
+		//Option("", options.Events).
+		Option("fscache", options.FsCache).
+		Option("hidden", options.Hidden).
+		Option("inline", options.Inline).
+		Option("inline-limit", options.InlineLimit).
+		Option("nocopy", options.NoCopy).
+		Option("only-hash", options.OnlyHash).
+		Option("pin", options.Pin).
+		//Option("", options.Progress).
+		Option("silent", options.Silent).
+		Option("stdin-name", options.StdinName).
+		Option("wrap-with-directory", options.Wrap).
+		Option("quieter", true) // TODO: rm after event impl
+
+	if options.RawLeavesSet {
+		req.Option("raw-leaves", options.RawLeaves)
+	}
+
+	switch options.Layout {
+	case caopts.BalancedLayout:
+		// noop, default
+	case caopts.TrickleLayout:
+		req.Option("trickle", true)
+	}
+
+	switch c := f.(type) {
+	case files.Directory:
+		req.Body(files.NewMultiFileReader(c, false))
+	case files.File:
+		req.Body(c)
+	}
+
+	var out addEvent
+	if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
+}
+
+func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) {
+	panic("implement me")
+}
+
+func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) {
+	panic("implement me")
+}
+
+func (api *UnixfsAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From cf74d391603457ad626b7f6b27ff0655bc946ee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 02:07:21 +0100
Subject: [PATCH 08/64] Partial Key API, ApiAddr funcion

This commit was moved from ipfs/go-ipfs-http-client@0ffdef159261e19cfd76edabb394179aa9295003
---
 client/httpapi/api.go | 13 ++++++-
 client/httpapi/key.go | 86 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100644 client/httpapi/key.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 099b45123..d104d98df 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -29,6 +29,7 @@ type HttpApi struct {
 	httpcli *gohttp.Client
 }
 
+//TODO: Return errors here
 func NewLocalApi() iface.CoreAPI {
 	baseDir := os.Getenv(EnvDir)
 	if baseDir == "" {
@@ -39,6 +40,14 @@ func NewLocalApi() iface.CoreAPI {
 }
 
 func NewPathApi(p string) iface.CoreAPI {
+	a := ApiAddr(p)
+	if a == nil {
+		return nil
+	}
+	return NewApi(a)
+}
+
+func ApiAddr(p string) ma.Multiaddr {
 	baseDir, err := homedir.Expand(p)
 	if err != nil {
 		return nil
@@ -60,7 +69,7 @@ func NewPathApi(p string) iface.CoreAPI {
 		return nil
 	}
 
-	return NewApi(maddr)
+	return maddr
 }
 
 func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr?
@@ -122,7 +131,7 @@ func (api *HttpApi) Name() iface.NameAPI {
 }
 
 func (api *HttpApi) Key() iface.KeyAPI {
-	return nil
+	return (*KeyAPI)(api)
 }
 
 func (api *HttpApi) Pin() iface.PinAPI {
diff --git a/client/httpapi/key.go b/client/httpapi/key.go
new file mode 100644
index 000000000..87b573f98
--- /dev/null
+++ b/client/httpapi/key.go
@@ -0,0 +1,86 @@
+package httpapi
+
+import (
+	"context"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/libp2p/go-libp2p-peer"
+)
+
+type KeyAPI HttpApi
+
+type keyOutput struct {
+	JName string `json:"Name"`
+	Id   string
+}
+
+func (k *keyOutput) Name() string {
+	return k.JName
+}
+
+func (k *keyOutput) Path() iface.Path {
+	p, _ := iface.ParsePath("/ipns/" + k.Id)
+	return p
+}
+
+func (k *keyOutput) ID() peer.ID {
+	p, _ := peer.IDB58Decode(k.Id)
+	return p
+}
+
+func (k *keyOutput) valid() error {
+	_, err := peer.IDB58Decode(k.Id)
+	return err
+}
+
+
+func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) {
+	options, err := caopts.KeyGenerateOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out keyOutput
+	err = api.core().request("key/gen", name).
+		Option("type", options.Algorithm).
+		Option("size", options.Size).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+	if err := out.valid(); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) {
+	panic("implement me")
+}
+
+func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
+	panic("implement me")
+}
+
+func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
+	var id struct{ID string}
+	if err := api.core().request("id").Exec(ctx, &id); err != nil {
+		return nil, err
+	}
+
+	out := keyOutput{JName: "self", Id: id.ID}
+	if err := out.valid(); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) {
+	panic("implement me")
+}
+
+func (api *KeyAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From c236393733bf3491c45002d67189ba78303332a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 02:08:18 +0100
Subject: [PATCH 09/64] Connect test swarms, don't compress api calls

This commit was moved from ipfs/go-ipfs-http-client@c6472d9b8286c932492db4d6bf03f890bf97759c
---
 client/httpapi/api_test.go | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index e32c456e5..02c7830bb 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	gohttp "net/http"
 	"os"
 	"path"
 	"strconv"
@@ -50,6 +51,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
+	if n > 1 {
+		connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"}
+		if err := c.Run(connectArgs); err != nil {
+			return nil, err
+		}
+	}
+
 	go func() {
 		<-ctx.Done()
 
@@ -80,7 +88,18 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 			return nil, err
 		}
 
-		apis[i] = NewPathApi(pth)
+		a := ApiAddr(pth)
+		if a == nil {
+			return nil, fmt.Errorf("nil addr for node")
+		}
+		c := &gohttp.Client{
+			Transport: &gohttp.Transport{
+				Proxy:             gohttp.ProxyFromEnvironment,
+				DisableKeepAlives: true,
+				DisableCompression: true,
+			},
+		}
+		apis[i] = NewApiWithClient(a, c)
 	}
 
 	return apis, nil

From 7861315f49398e20e7e1189671e42d0a16adbef0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 02:09:00 +0100
Subject: [PATCH 10/64] Wrap single files in Unixfs.Add

This commit was moved from ipfs/go-ipfs-http-client@16f77b24a1b150e7ec624d5675ded478ac6cb853
---
 client/httpapi/unixfs.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 567f60284..21f75f85a 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -66,7 +66,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	case files.Directory:
 		req.Body(files.NewMultiFileReader(c, false))
 	case files.File:
-		req.Body(c)
+		d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side
+		req.Body(files.NewMultiFileReader(d, false))
 	}
 
 	var out addEvent

From f638bae3a960a74d284503c30086f0bbffbe73c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 02:09:17 +0100
Subject: [PATCH 11/64] implement .Name

This commit was moved from ipfs/go-ipfs-http-client@e19e5f54e48d0909796add6d57e3e74e84eba85e
---
 client/httpapi/name.go | 72 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 9 deletions(-)

diff --git a/client/httpapi/name.go b/client/httpapi/name.go
index 7315ac2c3..fbc440b96 100644
--- a/client/httpapi/name.go
+++ b/client/httpapi/name.go
@@ -2,26 +2,80 @@ package httpapi
 
 import (
 	"context"
+	"fmt"
+	"github.com/ipfs/go-ipfs/namesys/opts"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 )
 
 type NameAPI HttpApi
 
-func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) {
+type ipnsEntry struct {
+	JName  string `json:"Name"`
+	JValue string `json:"Value"`
+}
+
+func (e *ipnsEntry) valid() (iface.Path, error) {
+	return iface.ParsePath(e.JValue)
+}
+
+func (e *ipnsEntry) Name() string {
+	return e.JName
+}
+
+func (e *ipnsEntry) Value() iface.Path {
+	p, _ := e.valid()
+	return p
+}
+
+func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) {
+	options, err := caopts.NamePublishOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	req := api.core().request("name/publish", p.String()).
+		Option("key", options.Key).
+		Option("allow-offline", options.AllowOffline).
+		Option("lifetime", options.ValidTime.String()).
+		Option("resolve", false)
+
+	if options.TTL != nil {
+		req.Option("ttl", options.TTL.String())
+	}
+
+	var out ipnsEntry
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	if _, err := out.valid(); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) {
 	return nil, ErrNotImplemented
 }
 
-func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) {
-	return nil, ErrNotImplemented
-}
+func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) {
+	options, err := caopts.NameResolveOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
 
-func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) {
-	// TODO: options!
+	ropts := nsopts.ProcessOpts(options.ResolveOpts)
+	if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 {
+		return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit)
+	}
 
-	req := api.core().request("name/resolve")
-	req.Arguments(name)
+	req := api.core().request("name/resolve", name).
+		Option("nocache", !options.Cache).
+		Option("recursive", ropts.Depth != 1).
+		Option("dht-record-count", ropts.DhtRecordCount).
+		Option("dht-timeout", ropts.DhtTimeout.String())
 
 	var out struct{ Path string }
 	if err := req.Exec(ctx, &out); err != nil {

From 1acf4163902ae3b9e0e182761deafee5e67415ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 02:53:24 +0100
Subject: [PATCH 12/64] Implement .Unixfs.Ls()

This commit was moved from ipfs/go-ipfs-http-client@eb1944fae32f4ccc43969ecd632681c2f2cbfa00
---
 client/httpapi/response.go | 14 ++++++++++++-
 client/httpapi/unixfs.go   | 43 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 27709769b..5749ca29e 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -33,7 +33,19 @@ func (r *Response) Decode(dec interface{}) error {
 		return r.Error
 	}
 
-	return json.NewDecoder(r.Output).Decode(dec)
+	n := 0
+	var err error
+	for {
+		err = json.NewDecoder(r.Output).Decode(dec)
+		if err != nil {
+			break
+		}
+		n++
+	}
+	if n > 0 && err == io.EOF {
+		err = nil
+	}
+	return err
 }
 
 type Error struct {
diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 21f75f85a..75e565a4a 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/ipfs/go-cid"
+	"github.com/pkg/errors"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
@@ -11,6 +12,7 @@ import (
 	"github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+	unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb"
 )
 
 type addEvent struct {
@@ -87,8 +89,45 @@ func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) {
 	panic("implement me")
 }
 
-func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) {
-	panic("implement me")
+type lsLink struct {
+	Name, Hash string
+	Size       uint64
+	Type       unixfspb.Data_DataType
+}
+
+type lsObject struct {
+	Hash  string
+	Links []lsLink
+}
+
+type lsOutput struct {
+	Objects []lsObject
+}
+
+func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) {
+	var out lsOutput
+	err := api.core().request("ls", p.String()).Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(out.Objects) != 1 {
+		return nil, errors.New("unexpected objects len")
+	}
+
+	links := make([]*format.Link, len(out.Objects[0].Links))
+	for i, l := range out.Objects[0].Links {
+		c, err := cid.Parse(l.Hash)
+		if err != nil {
+			return nil, err
+		}
+		links[i] = &format.Link{
+			Name: l.Name,
+			Size: l.Size,
+			Cid: c,
+		}
+	}
+	return links, nil
 }
 
 func (api *UnixfsAPI) core() *HttpApi {

From a6636aac599a1a8a4802ccadb885909f0ab88629 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 03:18:21 +0100
Subject: [PATCH 13/64] Imprement partian Pin API

This commit was moved from ipfs/go-ipfs-http-client@dbf90eac67d95722e88056ca16fa769a32dcd48f
---
 client/httpapi/api.go      |  2 +-
 client/httpapi/api_test.go | 16 +++++++-
 client/httpapi/pin.go      | 78 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 94 insertions(+), 2 deletions(-)
 create mode 100644 client/httpapi/pin.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index d104d98df..d5741da08 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -135,7 +135,7 @@ func (api *HttpApi) Key() iface.KeyAPI {
 }
 
 func (api *HttpApi) Pin() iface.PinAPI {
-	return nil
+	return (*PinAPI)(api)
 }
 
 func (api *HttpApi) Object() iface.ObjectAPI {
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 02c7830bb..5fdf18352 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 	"github.com/ipfs/go-ipfs/core/coreapi/interface/tests"
 
 	local "github.com/ipfs/iptb-plugins/local"
@@ -39,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
-	c := cli.NewCli()
+	c := cli.NewCli() //TODO: is there a better way?
 
 	initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)}
 	if err := c.Run(initArgs); err != nil {
@@ -100,6 +101,19 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 			},
 		}
 		apis[i] = NewApiWithClient(a, c)
+
+		// node cleanup
+		// TODO: pass --empty-repo somehow (how?)
+		pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive())
+		if err != nil {
+			return nil, err
+		}
+		for _, pin := range pins { //TODO: parallel
+			if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil {
+				return nil, err
+			}
+		}
+
 	}
 
 	return apis, nil
diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
new file mode 100644
index 000000000..02dcc4222
--- /dev/null
+++ b/client/httpapi/pin.go
@@ -0,0 +1,78 @@
+package httpapi
+
+import (
+	"context"
+	"github.com/ipfs/go-cid"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+)
+
+type PinAPI HttpApi
+
+type pinRefKeyObject struct {
+	Type string
+}
+
+type pinRefKeyList struct {
+	Keys map[string]pinRefKeyObject
+}
+
+type pin struct {
+	path iface.ResolvedPath
+	typ string
+}
+
+func (p *pin) Path() iface.ResolvedPath {
+	return p.path
+}
+
+func (p *pin) Type() string {
+	return p.typ
+}
+
+
+func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error {
+	panic("implement me")
+}
+
+func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) {
+	options, err := caopts.PinLsOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out pinRefKeyList
+	err = api.core().request("pin/ls").
+		Option("type", options.Type).Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	pins := make([]iface.Pin, 0, len(out.Keys))
+	for hash, p := range out.Keys {
+		c, err := cid.Parse(hash)
+		if err != nil {
+			return nil, err
+		}
+		pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)})
+	}
+
+	return pins, nil
+}
+
+func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error {
+	return api.core().request("pin/rm", p.String()).Exec(ctx, nil)
+}
+
+func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error {
+	panic("implement me")
+}
+
+func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) {
+	panic("implement me")
+}
+
+func (api *PinAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From 0f7c83956b2d769c801c270fe01d4819cb070881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 03:20:04 +0100
Subject: [PATCH 14/64] Import missing unixfs dep

This commit was moved from ipfs/go-ipfs-http-client@6169321d1df6495cbd1c06cca9f2e0777a00f638
---
 client/httpapi/unixfs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 75e565a4a..5a1495c24 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -11,8 +11,8 @@ import (
 
 	"github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipld-format"
+	unixfspb "github.com/ipfs/go-unixfs/pb"
 	mh "github.com/multiformats/go-multihash"
-	unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb"
 )
 
 type addEvent struct {

From af197cb6d90da1b51525e68f97a30ebfc322e134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 14:46:52 +0100
Subject: [PATCH 15/64] api.WithOption

This commit was moved from ipfs/go-ipfs-http-client@634b00bf1a42dc7f2a2dc75f85c77e5bb5ef727c
---
 client/httpapi/api.go            | 21 ++++++++++++++++++---
 client/httpapi/api_test.go       |  4 ++--
 client/httpapi/requestbuilder.go |  2 ++
 client/httpapi/response.go       |  2 +-
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index d5741da08..3df45f38c 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
@@ -27,6 +27,8 @@ var ErrNotImplemented = errors.New("not implemented")
 type HttpApi struct {
 	url     string
 	httpcli *gohttp.Client
+
+	applyGlobal func(*RequestBuilder)
 }
 
 //TODO: Return errors here
@@ -99,11 +101,24 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
 	return &HttpApi{
 		url:     url,
 		httpcli: c,
+		applyGlobal: func(*RequestBuilder) {},
 	}
 }
 
-func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) {
-	return nil, ErrNotImplemented
+func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) {
+	options, err := caopts.ApiOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	subApi := *api
+	subApi.applyGlobal = func(req *RequestBuilder) {
+		if options.Offline {
+			req.Option("offline", options.Offline)
+		}
+	}
+
+	return &subApi, nil
 }
 
 func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 5fdf18352..67a2dbdce 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -11,7 +11,7 @@ import (
 	"testing"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 	"github.com/ipfs/go-ipfs/core/coreapi/interface/tests"
 
 	local "github.com/ipfs/iptb-plugins/local"
@@ -104,7 +104,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 
 		// node cleanup
 		// TODO: pass --empty-repo somehow (how?)
-		pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive())
+		pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive())
 		if err != nil {
 			return nil, err
 		}
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index 6e5a89ebd..831e6d71c 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -86,6 +86,8 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder {
 
 // Send sends the request and return the response.
 func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) {
+	r.shell.applyGlobal(r)
+
 	req := NewRequest(ctx, r.shell.url, r.command, r.args...)
 	req.Opts = r.opts
 	req.Headers = r.headers
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 5749ca29e..f6e7f3ab7 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -21,7 +21,7 @@ type Response struct {
 func (r *Response) Close() error {
 	if r.Output != nil {
 		// always drain output (response body)
-		ioutil.ReadAll(r.Output)
+		//ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data
 		return r.Output.Close()
 	}
 	return nil

From 6d9dea62825012678ab0f117ec3cfe28c96fdaf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 8 Jan 2019 17:47:22 +0100
Subject: [PATCH 16/64] Implement Unixfs.Get()

This commit was moved from ipfs/go-ipfs-http-client@b31bee083d1fc5002779d755b6a624b517d2b10c
---
 client/httpapi/apifile.go | 231 ++++++++++++++++++++++++++++++++++++++
 client/httpapi/unixfs.go  |   4 -
 2 files changed, 231 insertions(+), 4 deletions(-)
 create mode 100644 client/httpapi/apifile.go

diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
new file mode 100644
index 000000000..541189b60
--- /dev/null
+++ b/client/httpapi/apifile.go
@@ -0,0 +1,231 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/ipfs/go-cid"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+
+	"github.com/ipfs/go-ipfs-files"
+	unixfspb "github.com/ipfs/go-unixfs/pb"
+)
+
+func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) {
+	if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS
+		var err error
+		p, err = api.core().ResolvePath(ctx, p)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var stat struct{
+		Hash string
+		Type string
+		Size int64 // unixfs size
+	}
+	err := api.core().request("files/stat", p.String()).	Exec(ctx, &stat)
+	if err != nil {
+		return nil, err
+	}
+
+	switch stat.Type {
+	case "file":
+		return api.getFile(ctx, p, stat.Size)
+	case "directory":
+		return api.getDir(ctx, p, stat.Size)
+	default:
+		return nil, fmt.Errorf("unsupported file type '%s'", stat.Type)
+	}
+}
+
+type apiFile struct {
+	ctx context.Context
+	core *HttpApi
+	size int64
+	path iface.Path
+
+	r io.ReadCloser
+	at int64
+}
+
+func (f *apiFile) reset() error {
+	if f.r != nil {
+		f.r.Close()
+	}
+	req := f.core.request("cat", f.path.String())
+	if f.at != 0 {
+		req.Option("offset", f.at)
+	}
+	resp, err := req.Send(f.ctx)
+	if err != nil {
+		return err
+	}
+	if resp.Error != nil {
+		return resp.Error
+	}
+	f.r = resp.Output
+	return nil
+}
+
+func (f *apiFile) Read(p []byte) (int, error) {
+	n, err := f.r.Read(p)
+	if n > 0 {
+		f.at += int64(n)
+	}
+	return n, err
+}
+
+func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
+	panic("implement me") //TODO
+}
+
+func (f *apiFile) Close() error {
+	if f.r != nil {
+		return f.r.Close()
+	}
+	return nil
+}
+
+func (f *apiFile) Size() (int64, error) {
+	return f.size, nil
+}
+
+func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) {
+	f := &apiFile{
+		ctx: ctx,
+		core: api.core(),
+		size: size,
+		path: p,
+	}
+
+	return f, f.reset()
+}
+
+type apiIter struct {
+	ctx context.Context
+	core *UnixfsAPI
+
+	err error
+
+	dec *json.Decoder
+	curFile files.Node
+	cur lsLink
+}
+
+func (it *apiIter) Err() error {
+	return it.err
+}
+
+func (it *apiIter) Name() string {
+	return it.cur.Name
+}
+
+func (it *apiIter) Next() bool {
+	var out lsOutput
+	if err := it.dec.Decode(&out); err != nil {
+		if err != io.EOF {
+			it.err = err
+		}
+		return false
+	}
+
+	if len(out.Objects) != 1 {
+		it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects))
+		return false
+	}
+
+	if len(out.Objects[0].Links) != 1 {
+		it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links))
+		return false
+	}
+
+	it.cur = out.Objects[0].Links[0]
+	c, err := cid.Parse(it.cur.Hash)
+	if err != nil {
+		it.err = err
+		return false
+	}
+
+	switch it.cur.Type {
+	case unixfspb.Data_HAMTShard:
+		fallthrough
+	case unixfspb.Data_Metadata:
+		fallthrough
+	case unixfspb.Data_Directory:
+		it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size))
+		if err != nil {
+			it.err = err
+			return false
+		}
+	case unixfspb.Data_File:
+		it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size))
+		if err != nil {
+			it.err = err
+			return false
+		}
+	default:
+		it.err = fmt.Errorf("file type %d not supported", it.cur.Type)
+		return false
+	}
+	return true
+}
+
+func (it *apiIter) Node() files.Node {
+	return it.curFile
+}
+
+type apiDir struct {
+	ctx context.Context
+	core *UnixfsAPI
+	size int64
+	path iface.Path
+
+	dec *json.Decoder
+}
+
+func (d *apiDir) Close() error {
+	return nil
+}
+
+func (d *apiDir) Size() (int64, error) {
+	return d.size, nil
+}
+
+func (d *apiDir) Entries() files.DirIterator {
+	return &apiIter{
+		ctx: d.ctx,
+		core: d.core,
+		dec: d.dec,
+	}
+}
+
+func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) {
+	resp, err := api.core().request("ls", p.String()).
+		Option("resolve-size", true).
+		Option("stream", true).Send(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	d := &apiDir{
+		ctx: ctx,
+		core: api,
+		size: size,
+		path: p,
+
+		dec: json.NewDecoder(resp.Output),
+	}
+
+	return d, nil
+}
+
+var _ files.File = &apiFile{}
+var _ files.Directory = &apiDir{}
diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 5a1495c24..77daf7b9b 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -85,10 +85,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	return iface.IpfsPath(c), nil
 }
 
-func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) {
-	panic("implement me")
-}
-
 type lsLink struct {
 	Name, Hash string
 	Size       uint64

From 00597e6dff81846a2dbf742dd1ab9bf546b856e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 9 Jan 2019 09:04:22 +0100
Subject: [PATCH 17/64] Dag.Put

This commit was moved from ipfs/go-ipfs-http-client@3217104469020f0c66d8457842f8221b64289d74
---
 client/httpapi/api.go |  2 +-
 client/httpapi/dag.go | 69 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 client/httpapi/dag.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 3df45f38c..c2ec1ad67 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -138,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI {
 }
 
 func (api *HttpApi) Dag() iface.DagAPI {
-	return nil
+	return (*DagAPI)(api)
 }
 
 func (api *HttpApi) Name() iface.NameAPI {
diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
new file mode 100644
index 000000000..c3b29e7ee
--- /dev/null
+++ b/client/httpapi/dag.go
@@ -0,0 +1,69 @@
+package httpapi
+
+import (
+	"context"
+	"fmt"
+	"github.com/ipfs/go-cid"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	"github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type DagAPI HttpApi
+
+func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) {
+	options, err := caopts.DagPutOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	mht, ok := mh.Codes[options.MhType]
+	if !ok {
+		return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+	}
+
+	codec, ok := cid.CodecToStr[options.Codec]
+	if !ok {
+		return nil, fmt.Errorf("unknowm codec %d", options.MhType)
+	}
+
+	if options.MhLength != -1 {
+		return nil, fmt.Errorf("setting hash len is not supported yet")
+	}
+
+	var out struct{
+		Cid cid.Cid
+	}
+	err = api.core().request("dht/put").
+		Option("hash", mht).
+		Option("format", codec).
+		Option("input-enc", options.InputEnc).
+		FileBody(src).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpldPath(out.Cid), nil
+}
+
+func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) {
+	panic("implement me")
+}
+
+func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) {
+	panic("implement me")
+}
+
+func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch {
+	panic("implement me")
+}
+
+func (api *DagAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From 101910f04ed0575be19efaacb44b1bce73717033 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 9 Jan 2019 09:22:10 +0100
Subject: [PATCH 18/64] Object.New

This commit was moved from ipfs/go-ipfs-http-client@60321ed42fd8f42e964e37ab5e96839fb8fd77b6
---
 client/httpapi/api.go      |  2 +-
 client/httpapi/ipldnode.go |  8 ++++
 client/httpapi/object.go   | 82 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 client/httpapi/object.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index c2ec1ad67..cf9a633bc 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -154,7 +154,7 @@ func (api *HttpApi) Pin() iface.PinAPI {
 }
 
 func (api *HttpApi) Object() iface.ObjectAPI {
-	return nil
+	return (*ObjectAPI)(api)
 }
 
 func (api *HttpApi) Dht() iface.DhtAPI {
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
index b8e6fba01..1f584ac07 100644
--- a/client/httpapi/ipldnode.go
+++ b/client/httpapi/ipldnode.go
@@ -20,6 +20,14 @@ type ipldNode struct {
 	api  *HttpApi
 }
 
+func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node {
+	return &ipldNode{
+		ctx: ctx,
+		path: p,
+		api: a,
+	}
+}
+
 func (n *ipldNode) RawData() []byte {
 	r, err := n.api.Block().Get(n.ctx, n.path)
 	if err != nil {
diff --git a/client/httpapi/object.go b/client/httpapi/object.go
new file mode 100644
index 000000000..9c1063af9
--- /dev/null
+++ b/client/httpapi/object.go
@@ -0,0 +1,82 @@
+package httpapi
+
+import (
+	"context"
+	"github.com/ipfs/go-cid"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/ipfs/go-ipld-format"
+)
+
+type ObjectAPI HttpApi
+
+type objectOut struct {
+	Hash  string
+}
+
+func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) {
+	options, err := caopts.ObjectNewOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out objectOut
+	err = api.core().request("object/new", options.Type).Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil
+}
+
+func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) {
+	panic("implement me")
+}
+
+func (api *ObjectAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From 31a4c3754b87e68f60525533ba9c6bb9c0b1021e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 9 Jan 2019 09:47:32 +0100
Subject: [PATCH 19/64] Fix Dag.Put

This commit was moved from ipfs/go-ipfs-http-client@5b2c99abdedaf464f061511bc7b24b1ade66546f
---
 client/httpapi/dag.go | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index c3b29e7ee..152d2c6b0 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -3,30 +3,25 @@ package httpapi
 import (
 	"context"
 	"fmt"
-	"github.com/ipfs/go-cid"
 	"io"
+	"math"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
 )
 
 type DagAPI HttpApi
 
-func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) {
+func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) {
 	options, err := caopts.DagPutOptions(opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	mht, ok := mh.Codes[options.MhType]
-	if !ok {
-		return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
-	}
-
 	codec, ok := cid.CodecToStr[options.Codec]
 	if !ok {
 		return nil, fmt.Errorf("unknowm codec %d", options.MhType)
@@ -39,12 +34,19 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPu
 	var out struct{
 		Cid cid.Cid
 	}
-	err = api.core().request("dht/put").
-		Option("hash", mht).
+	req := api.core().request("dag/put").
 		Option("format", codec).
-		Option("input-enc", options.InputEnc).
-		FileBody(src).
-		Exec(ctx, &out)
+		Option("input-enc", options.InputEnc)
+
+	if options.MhType != math.MaxUint64 {
+		mht, ok := mh.Codes[options.MhType]
+		if !ok {
+			return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+		}
+		req.Option("hash", mht)
+	}
+
+	err = req.FileBody(src).Exec(ctx, &out)
 	if err != nil {
 		return nil, err
 	}
@@ -56,7 +58,7 @@ func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error
 	panic("implement me")
 }
 
-func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) {
+func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) {
 	panic("implement me")
 }
 

From c213e2654235ffbdb6ce2f3a7df3c027ba53e02f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 9 Jan 2019 21:55:45 +0100
Subject: [PATCH 20/64] Unixfs.Add progress events

This commit was moved from ipfs/go-ipfs-http-client@2f3a77b686ee8c4194ee6c21cbdecd57f2081519
---
 client/httpapi/api.go      |  4 +--
 client/httpapi/api_test.go |  6 ++---
 client/httpapi/apifile.go  | 24 +++++++++---------
 client/httpapi/block.go    |  2 +-
 client/httpapi/dag.go      |  2 +-
 client/httpapi/ipldnode.go |  4 +--
 client/httpapi/key.go      |  5 ++--
 client/httpapi/object.go   |  2 +-
 client/httpapi/pin.go      |  3 +--
 client/httpapi/unixfs.go   | 51 ++++++++++++++++++++++++++++++++++----
 10 files changed, 71 insertions(+), 32 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index cf9a633bc..c686a8d0d 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -99,8 +99,8 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
 	}
 
 	return &HttpApi{
-		url:     url,
-		httpcli: c,
+		url:         url,
+		httpcli:     c,
 		applyGlobal: func(*RequestBuilder) {},
 	}
 }
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 67a2dbdce..b822378f7 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -53,7 +53,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 	}
 
 	if n > 1 {
-		connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"}
+		connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"}
 		if err := c.Run(connectArgs); err != nil {
 			return nil, err
 		}
@@ -95,8 +95,8 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		}
 		c := &gohttp.Client{
 			Transport: &gohttp.Transport{
-				Proxy:             gohttp.ProxyFromEnvironment,
-				DisableKeepAlives: true,
+				Proxy:              gohttp.ProxyFromEnvironment,
+				DisableKeepAlives:  true,
 				DisableCompression: true,
 			},
 		}
diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index 541189b60..d9da23975 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -22,12 +22,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error)
 		}
 	}
 
-	var stat struct{
+	var stat struct {
 		Hash string
 		Type string
 		Size int64 // unixfs size
 	}
-	err := api.core().request("files/stat", p.String()).	Exec(ctx, &stat)
+	err := api.core().request("files/stat", p.String()).Exec(ctx, &stat)
 	if err != nil {
 		return nil, err
 	}
@@ -43,12 +43,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error)
 }
 
 type apiFile struct {
-	ctx context.Context
+	ctx  context.Context
 	core *HttpApi
 	size int64
 	path iface.Path
 
-	r io.ReadCloser
+	r  io.ReadCloser
 	at int64
 }
 
@@ -96,7 +96,7 @@ func (f *apiFile) Size() (int64, error) {
 
 func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) {
 	f := &apiFile{
-		ctx: ctx,
+		ctx:  ctx,
 		core: api.core(),
 		size: size,
 		path: p,
@@ -106,14 +106,14 @@ func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (fi
 }
 
 type apiIter struct {
-	ctx context.Context
+	ctx  context.Context
 	core *UnixfsAPI
 
 	err error
 
-	dec *json.Decoder
+	dec     *json.Decoder
 	curFile files.Node
-	cur lsLink
+	cur     lsLink
 }
 
 func (it *apiIter) Err() error {
@@ -179,7 +179,7 @@ func (it *apiIter) Node() files.Node {
 }
 
 type apiDir struct {
-	ctx context.Context
+	ctx  context.Context
 	core *UnixfsAPI
 	size int64
 	path iface.Path
@@ -197,9 +197,9 @@ func (d *apiDir) Size() (int64, error) {
 
 func (d *apiDir) Entries() files.DirIterator {
 	return &apiIter{
-		ctx: d.ctx,
+		ctx:  d.ctx,
 		core: d.core,
-		dec: d.dec,
+		dec:  d.dec,
 	}
 }
 
@@ -216,7 +216,7 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (fil
 	}
 
 	d := &apiDir{
-		ctx: ctx,
+		ctx:  ctx,
 		core: api,
 		size: size,
 		path: p,
diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index 185aa0a42..f933d3c4a 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -16,7 +16,7 @@ import (
 type BlockAPI HttpApi
 
 type blockStat struct {
-	Key  string
+	Key   string
 	BSize int `json:"Size"`
 }
 
diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index 152d2c6b0..20c233196 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -31,7 +31,7 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPut
 		return nil, fmt.Errorf("setting hash len is not supported yet")
 	}
 
-	var out struct{
+	var out struct {
 		Cid cid.Cid
 	}
 	req := api.core().request("dag/put").
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
index 1f584ac07..8e028e1a7 100644
--- a/client/httpapi/ipldnode.go
+++ b/client/httpapi/ipldnode.go
@@ -22,9 +22,9 @@ type ipldNode struct {
 
 func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node {
 	return &ipldNode{
-		ctx: ctx,
+		ctx:  ctx,
 		path: p,
-		api: a,
+		api:  a,
 	}
 }
 
diff --git a/client/httpapi/key.go b/client/httpapi/key.go
index 87b573f98..3653526eb 100644
--- a/client/httpapi/key.go
+++ b/client/httpapi/key.go
@@ -13,7 +13,7 @@ type KeyAPI HttpApi
 
 type keyOutput struct {
 	JName string `json:"Name"`
-	Id   string
+	Id    string
 }
 
 func (k *keyOutput) Name() string {
@@ -35,7 +35,6 @@ func (k *keyOutput) valid() error {
 	return err
 }
 
-
 func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) {
 	options, err := caopts.KeyGenerateOptions(opts...)
 	if err != nil {
@@ -65,7 +64,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
 }
 
 func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
-	var id struct{ID string}
+	var id struct{ ID string }
 	if err := api.core().request("id").Exec(ctx, &id); err != nil {
 		return nil, err
 	}
diff --git a/client/httpapi/object.go b/client/httpapi/object.go
index 9c1063af9..6259473c7 100644
--- a/client/httpapi/object.go
+++ b/client/httpapi/object.go
@@ -14,7 +14,7 @@ import (
 type ObjectAPI HttpApi
 
 type objectOut struct {
-	Hash  string
+	Hash string
 }
 
 func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) {
diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index 02dcc4222..ea34dc540 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -20,7 +20,7 @@ type pinRefKeyList struct {
 
 type pin struct {
 	path iface.ResolvedPath
-	typ string
+	typ  string
 }
 
 func (p *pin) Path() iface.ResolvedPath {
@@ -31,7 +31,6 @@ func (p *pin) Type() string {
 	return p.typ
 }
 
-
 func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error {
 	panic("implement me")
 }
diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 77daf7b9b..7e63d241f 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -2,9 +2,11 @@ package httpapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"github.com/ipfs/go-cid"
 	"github.com/pkg/errors"
+	"io"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
@@ -39,7 +41,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 		Option("hash", mht).
 		Option("chunker", options.Chunker).
 		Option("cid-version", options.CidVersion).
-		//Option("", options.Events).
 		Option("fscache", options.FsCache).
 		Option("hidden", options.Hidden).
 		Option("inline", options.Inline).
@@ -47,11 +48,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 		Option("nocopy", options.NoCopy).
 		Option("only-hash", options.OnlyHash).
 		Option("pin", options.Pin).
-		//Option("", options.Progress).
 		Option("silent", options.Silent).
 		Option("stdin-name", options.StdinName).
 		Option("wrap-with-directory", options.Wrap).
-		Option("quieter", true) // TODO: rm after event impl
+		Option("progress", options.Progress)
 
 	if options.RawLeavesSet {
 		req.Option("raw-leaves", options.RawLeaves)
@@ -73,9 +73,50 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	}
 
 	var out addEvent
-	if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events
+	resp, err := req.Send(ctx)
+	if err != nil {
 		return nil, err
 	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	defer resp.Output.Close()
+	dec := json.NewDecoder(resp.Output)
+loop:
+	for {
+		var evt addEvent
+		switch err := dec.Decode(&evt); err {
+		case nil:
+		case io.EOF:
+			break loop
+		default:
+			return nil, err
+		}
+		out = evt
+
+		if options.Events != nil {
+			ifevt := &iface.AddEvent{
+				Name:  out.Name,
+				Size:  out.Size,
+				Bytes: out.Bytes,
+			}
+
+			if out.Hash != "" {
+				c, err := cid.Parse(out.Hash)
+				if err != nil {
+					return nil, err
+				}
+
+				ifevt.Path = iface.IpfsPath(c)
+			}
+
+			select {
+			case options.Events <- ifevt:
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		}
+	}
 
 	c, err := cid.Parse(out.Hash)
 	if err != nil {
@@ -120,7 +161,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, err
 		links[i] = &format.Link{
 			Name: l.Name,
 			Size: l.Size,
-			Cid: c,
+			Cid:  c,
 		}
 	}
 	return links, nil

From 266c2f92c50df9b50bbc5aa093c793bb5d1a2265 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 14:33:03 +0100
Subject: [PATCH 21/64] Implement Key API

This commit was moved from ipfs/go-ipfs-http-client@281b2bf65b19b17e4f0c047ac8f9901306bf171f
---
 client/httpapi/key.go | 57 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/client/httpapi/key.go b/client/httpapi/key.go
index 3653526eb..417415ddb 100644
--- a/client/httpapi/key.go
+++ b/client/httpapi/key.go
@@ -2,6 +2,7 @@ package httpapi
 
 import (
 	"context"
+	"errors"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
@@ -49,18 +50,47 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key
 	if err != nil {
 		return nil, err
 	}
-	if err := out.valid(); err != nil {
-		return nil, err
-	}
-	return &out, nil
+	return &out, out.valid()
 }
 
 func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) {
-	panic("implement me")
+	options, err := caopts.KeyRenameOptions(opts...)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var out struct{
+		Was string
+		Now string
+		Id string
+		Overwrite bool
+	}
+	err = api.core().request("key/rename", oldName, newName).
+		Option("force", options.Force).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, false, err
+	}
+
+	id := &keyOutput{JName: out.Now, Id: out.Id}
+	return id, out.Overwrite, id.valid()
 }
 
 func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
-	panic("implement me")
+	var out struct{ Keys []*keyOutput }
+	if err := api.core().request("key/list").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]iface.Key, len(out.Keys))
+	for i, k := range out.Keys {
+		if err := k.valid(); err != nil {
+			return nil, err
+		}
+		res[i] = k
+	}
+
+	return res, nil
 }
 
 func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
@@ -70,14 +100,19 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
 	}
 
 	out := keyOutput{JName: "self", Id: id.ID}
-	if err := out.valid(); err != nil {
-		return nil, err
-	}
-	return &out, nil
+	return &out, out.valid()
 }
 
 func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) {
-	panic("implement me")
+	var out struct{ Keys []keyOutput }
+	if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	if len(out.Keys) != 1 {
+		return nil, errors.New("got unexpected number of keys back")
+	}
+
+	return &out.Keys[0], out.Keys[0].valid()
 }
 
 func (api *KeyAPI) core() *HttpApi {

From 9d647d011efa551bae28c50cafb402a35d33b04a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 15:02:15 +0100
Subject: [PATCH 22/64] Implement Pin API

This commit was moved from ipfs/go-ipfs-http-client@5bb7a581323aad4dd765a0916f6552cd5902d1e2
---
 client/httpapi/pin.go | 92 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 5 deletions(-)

diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index ea34dc540..34cafad71 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -2,7 +2,9 @@ package httpapi
 
 import (
 	"context"
+	"encoding/json"
 	"github.com/ipfs/go-cid"
+	"github.com/pkg/errors"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
@@ -31,8 +33,14 @@ func (p *pin) Type() string {
 	return p.typ
 }
 
-func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error {
-	panic("implement me")
+func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error {
+	options, err := caopts.PinAddOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().request("pin/add", p.String()).
+		Option("recursive", options.Recursive).Exec(ctx, nil)
 }
 
 func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) {
@@ -65,11 +73,85 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error {
 }
 
 func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error {
-	panic("implement me")
+	options, err := caopts.PinUpdateOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().request("pin/update").
+		Option("unpin", options.Unpin).Exec(ctx, nil)
 }
 
-func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) {
-	panic("implement me")
+type pinVerifyRes struct {
+	Cid      string
+	JOk       bool `json:"Ok"`
+	JBadNodes []*badNode `json:"BadNodes,omitempty"`
+}
+
+func (r *pinVerifyRes) Ok() bool {
+	return r.JOk
+}
+
+func (r *pinVerifyRes) BadNodes() []iface.BadPinNode {
+	out := make([]iface.BadPinNode, len(r.JBadNodes))
+	for i, n := range r.JBadNodes {
+		out[i] = n
+	}
+	return out
+}
+
+type badNode struct {
+	Cid string
+	JErr string `json:"Err"`
+}
+
+func (n *badNode) Path() iface.ResolvedPath {
+	c, err := cid.Parse(n.Cid)
+	if err != nil {
+		return nil // todo: handle this better
+	}
+	return iface.IpldPath(c)
+}
+
+func (n *badNode) Err() error {
+	if n.JErr != "" {
+		return errors.New(n.JErr)
+	}
+	if _, err := cid.Parse(n.Cid); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
+	resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	res := make(chan iface.PinStatus)
+
+	go func() {
+		defer resp.Close()
+		defer close(res)
+		dec := json.NewDecoder(resp.Output)
+		for {
+			var out pinVerifyRes
+			if err := dec.Decode(&out); err != nil {
+				return // todo: handle non io.EOF somehow
+			}
+
+			select {
+			case res <- &out:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return res, nil
 }
 
 func (api *PinAPI) core() *HttpApi {

From eaa19388d4d86eebf09d93a3a637e0690df722bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 16:31:07 +0100
Subject: [PATCH 23/64] Implement Object API

This commit was moved from ipfs/go-ipfs-http-client@38149e46c8c3a1544dc3e3108ef72ffbc885e410
---
 client/httpapi/block.go    |   2 +-
 client/httpapi/ipldnode.go |   7 +-
 client/httpapi/key.go      |   8 +-
 client/httpapi/object.go   | 216 +++++++++++++++++++++++++++++++++----
 client/httpapi/pin.go      |   6 +-
 5 files changed, 208 insertions(+), 31 deletions(-)

diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index f933d3c4a..f8334a109 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -67,7 +67,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP
 }
 
 func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) {
-	resp, err := api.core().request("block/get", p.String()).Send(context.Background())
+	resp, err := api.core().request("block/get", p.String()).Send(ctx)
 	if err != nil {
 		return nil, err
 	}
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
index 8e028e1a7..2294a95b4 100644
--- a/client/httpapi/ipldnode.go
+++ b/client/httpapi/ipldnode.go
@@ -3,7 +3,6 @@ package httpapi
 import (
 	"context"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"strconv"
 
@@ -20,11 +19,11 @@ type ipldNode struct {
 	api  *HttpApi
 }
 
-func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node {
+func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node {
 	return &ipldNode{
 		ctx:  ctx,
 		path: p,
-		api:  a,
+		api:  api,
 	}
 }
 
@@ -47,7 +46,7 @@ func (n *ipldNode) Cid() cid.Cid {
 }
 
 func (n *ipldNode) String() string {
-	return fmt.Sprintf("[Block %s]", n.Cid())
+	return n.Cid().String()
 }
 
 func (n *ipldNode) Loggable() map[string]interface{} {
diff --git a/client/httpapi/key.go b/client/httpapi/key.go
index 417415ddb..5087b6d99 100644
--- a/client/httpapi/key.go
+++ b/client/httpapi/key.go
@@ -59,10 +59,10 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o
 		return nil, false, err
 	}
 
-	var out struct{
-		Was string
-		Now string
-		Id string
+	var out struct {
+		Was       string
+		Now       string
+		Id        string
 		Overwrite bool
 	}
 	err = api.core().request("key/rename", oldName, newName).
diff --git a/client/httpapi/object.go b/client/httpapi/object.go
index 6259473c7..4bc787991 100644
--- a/client/httpapi/object.go
+++ b/client/httpapi/object.go
@@ -1,14 +1,17 @@
 package httpapi
 
 import (
+	"bytes"
 	"context"
-	"github.com/ipfs/go-cid"
 	"io"
+	"io/ioutil"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
 )
 
 type ObjectAPI HttpApi
@@ -37,44 +40,219 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (
 	return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil
 }
 
-func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) {
-	panic("implement me")
+func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) {
+	options, err := caopts.ObjectPutOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out objectOut
+	err = api.core().request("object/put").
+		Option("inputenc", options.InputEnc).
+		Option("datafieldenc", options.DataType).
+		Option("pin", options.Pin).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
 }
 
-func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) {
-	panic("implement me")
+func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) {
+	r, err := api.core().Block().Get(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return merkledag.DecodeProtobuf(b)
 }
 
-func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) {
-	panic("implement me")
+func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) {
+	resp, err := api.core().request("object/data", p.String()).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	//TODO: make Data return ReadCloser to avoid copying
+	defer resp.Close()
+	b := new(bytes.Buffer)
+	if _, err := io.Copy(b, resp.Output); err != nil {
+		return nil, err
+	}
+
+	return b, nil
 }
 
-func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) {
-	panic("implement me")
+func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) {
+	var out struct {
+		Links []struct {
+			Name string
+			Hash string
+			Size uint64
+		}
+	}
+	if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := make([]*format.Link, len(out.Links))
+	for i, l := range out.Links {
+		c, err := cid.Parse(l.Hash)
+		if err != nil {
+			return nil, err
+		}
+
+		res[i] = &format.Link{
+			Cid:  c,
+			Name: l.Name,
+			Size: l.Size,
+		}
+	}
+
+	return res, nil
 }
 
-func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) {
-	panic("implement me")
+func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) {
+	var out struct {
+		Hash           string
+		NumLinks       int
+		BlockSize      int
+		LinksSize      int
+		DataSize       int
+		CumulativeSize int
+	}
+	if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return &iface.ObjectStat{
+		Cid:            c,
+		NumLinks:       out.NumLinks,
+		BlockSize:      out.BlockSize,
+		LinksSize:      out.LinksSize,
+		DataSize:       out.DataSize,
+		CumulativeSize: out.CumulativeSize,
+	}, nil
 }
 
 func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) {
-	panic("implement me")
+	options, err := caopts.ObjectAddLinkOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out objectOut
+	err = api.core().request("object/patch/add-link", base.String(), name, child.String()).
+		Option("create", options.Create).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
 }
 
 func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) {
-	panic("implement me")
+	var out objectOut
+	err := api.core().request("object/patch/rm-link", base.String(), link).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
 }
 
-func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) {
-	panic("implement me")
+func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) {
+	var out objectOut
+	err := api.core().request("object/patch/append-data", p.String()).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
 }
 
-func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) {
-	panic("implement me")
+func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) {
+	var out objectOut
+	err := api.core().request("object/patch/set-data", p.String()).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return iface.IpfsPath(c), nil
 }
 
-func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) {
-	panic("implement me")
+type change struct {
+	Type   iface.ChangeType
+	Path   string
+	Before cid.Cid
+	After  cid.Cid
+}
+
+func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) {
+	var out struct {
+		Changes []change
+	}
+	if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := make([]iface.ObjectChange, len(out.Changes))
+	for i, ch := range out.Changes {
+		res[i] = iface.ObjectChange{
+			Type: ch.Type,
+			Path: ch.Path,
+		}
+		if ch.Before != cid.Undef {
+			res[i].Before = iface.IpfsPath(ch.Before)
+		}
+		if ch.After != cid.Undef {
+			res[i].After = iface.IpfsPath(ch.After)
+		}
+	}
+	return res, nil
 }
 
 func (api *ObjectAPI) core() *HttpApi {
diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index 34cafad71..1624b7867 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -83,8 +83,8 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o
 }
 
 type pinVerifyRes struct {
-	Cid      string
-	JOk       bool `json:"Ok"`
+	Cid       string
+	JOk       bool       `json:"Ok"`
 	JBadNodes []*badNode `json:"BadNodes,omitempty"`
 }
 
@@ -101,7 +101,7 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode {
 }
 
 type badNode struct {
-	Cid string
+	Cid  string
 	JErr string `json:"Err"`
 }
 

From ed9f2dd091693e914e9de072b2c23c2f991210a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 17:01:33 +0100
Subject: [PATCH 24/64] Implement DHT Api

This commit was moved from ipfs/go-ipfs-http-client@c77355067a22300a8abe6b93b354de0ae0a3b548
---
 client/httpapi/api.go |  2 +-
 client/httpapi/dht.go | 98 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 client/httpapi/dht.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index c686a8d0d..208b958e9 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -158,7 +158,7 @@ func (api *HttpApi) Object() iface.ObjectAPI {
 }
 
 func (api *HttpApi) Dht() iface.DhtAPI {
-	return nil
+	return (*DhtAPI)(api)
 }
 
 func (api *HttpApi) Swarm() iface.SwarmAPI {
diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go
new file mode 100644
index 000000000..17d3b0500
--- /dev/null
+++ b/client/httpapi/dht.go
@@ -0,0 +1,98 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/libp2p/go-libp2p-peer"
+	"github.com/libp2p/go-libp2p-peerstore"
+	notif "github.com/libp2p/go-libp2p-routing/notifications"
+)
+
+type DhtAPI HttpApi
+
+func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) {
+	var out struct {
+		Type      notif.QueryEventType
+		Responses []peerstore.PeerInfo
+	}
+	resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx)
+	if err != nil {
+		return peerstore.PeerInfo{}, err
+	}
+	if resp.Error != nil {
+		return peerstore.PeerInfo{}, resp.Error
+	}
+	defer resp.Close()
+	dec := json.NewDecoder(resp.Output)
+	for {
+		if err := dec.Decode(&out); err != nil {
+			return peerstore.PeerInfo{}, err
+		}
+		if out.Type == notif.FinalPeer {
+			return out.Responses[0], nil
+		}
+	}
+}
+
+func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) {
+	options, err := caopts.DhtFindProvidersOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := api.core().request("dht/findprovs", p.String()).
+		Option("num-providers", options.NumProviders).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	res := make(chan peerstore.PeerInfo)
+
+	go func() {
+		defer resp.Close()
+		defer close(res)
+		dec := json.NewDecoder(resp.Output)
+
+		for {
+			var out struct {
+				Type      notif.QueryEventType
+				Responses []peerstore.PeerInfo
+			}
+
+			if err := dec.Decode(&out); err != nil {
+				return // todo: handle this somehow
+			}
+			if out.Type == notif.Provider {
+				for _, pi := range out.Responses {
+					select {
+					case res <- pi:
+					case <-ctx.Done():
+						return
+					}
+				}
+			}
+		}
+	}()
+
+	return res, nil
+}
+
+func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error {
+	options, err := caopts.DhtProvideOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().request("dht/provide", p.String()).
+		Option("recursive", options.Recursive).
+		Exec(ctx, nil)
+}
+
+func (api *DhtAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From cae0ff2379f29a205426f7a8199c398885f33897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 17:36:35 +0100
Subject: [PATCH 25/64] Implement Swarm Api

This commit was moved from ipfs/go-ipfs-http-client@01105690d2da6e3f3581fa364c6284248a251a35
---
 client/httpapi/api.go   |   2 +-
 client/httpapi/swarm.go | 176 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 client/httpapi/swarm.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 208b958e9..a5b56eff6 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -162,7 +162,7 @@ func (api *HttpApi) Dht() iface.DhtAPI {
 }
 
 func (api *HttpApi) Swarm() iface.SwarmAPI {
-	return nil
+	return (*SwarmAPI)(api)
 }
 
 func (api *HttpApi) PubSub() iface.PubSubAPI {
diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
new file mode 100644
index 000000000..cf0ee2a9b
--- /dev/null
+++ b/client/httpapi/swarm.go
@@ -0,0 +1,176 @@
+package httpapi
+
+import (
+	"context"
+	"github.com/libp2p/go-libp2p-protocol"
+	"time"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+
+	inet "github.com/libp2p/go-libp2p-net"
+	"github.com/libp2p/go-libp2p-peer"
+	"github.com/libp2p/go-libp2p-peerstore"
+	"github.com/multiformats/go-multiaddr"
+)
+
+type SwarmAPI HttpApi
+
+func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error {
+	saddrs := make([]string, len(pi.Addrs))
+	for i, addr := range pi.Addrs {
+		saddrs[i] = addr.String()
+	}
+
+	return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil)
+}
+
+func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error {
+	return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil)
+}
+
+type streamInfo struct {
+	Protocol string
+}
+
+type connInfo struct {
+	Addr       string
+	Peer       string
+	JLatency   time.Duration `json:"Latency"`
+	Muxer      string
+	JDirection inet.Direction `json:"Direction"`
+	JStreams   []streamInfo   `json:"Streams"`
+}
+
+func (c *connInfo) valid() error {
+	_, err := multiaddr.NewMultiaddr(c.Addr)
+	if err != nil {
+		return err
+	}
+
+	_, err = peer.IDB58Decode(c.Peer)
+	return err
+}
+
+func (c *connInfo) ID() peer.ID {
+	id, _ := peer.IDB58Decode(c.Peer)
+	return id
+}
+
+func (c *connInfo) Address() multiaddr.Multiaddr {
+	a, _ := multiaddr.NewMultiaddr(c.Addr)
+	return a
+}
+
+func (c *connInfo) Direction() inet.Direction {
+	return c.JDirection
+}
+
+func (c *connInfo) Latency() (time.Duration, error) {
+	return c.JLatency, nil
+}
+
+func (c *connInfo) Streams() ([]protocol.ID, error) {
+	res := make([]protocol.ID, len(c.JStreams))
+	for i, stream := range c.JStreams {
+		res[i] = protocol.ID(stream.Protocol)
+	}
+	return res, nil
+}
+
+func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) {
+	var out struct {
+		Peers []*connInfo
+	}
+
+	err := api.core().request("swarm/peers").
+		Option("streams", true).
+		Option("latency", true).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]iface.ConnectionInfo, len(out.Peers))
+	for i, conn := range out.Peers {
+		if err := conn.valid(); err != nil {
+			return nil, err
+		}
+		res[i] = conn
+	}
+
+	return res, nil
+}
+
+func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) {
+	var out struct {
+		Addrs map[string][]string
+	}
+	if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := map[peer.ID][]multiaddr.Multiaddr{}
+	for spid, saddrs := range out.Addrs {
+		addrs := make([]multiaddr.Multiaddr, len(saddrs))
+
+		for i, addr := range saddrs {
+			a, err := multiaddr.NewMultiaddr(addr)
+			if err != nil {
+				return nil, err
+			}
+			addrs[i] = a
+		}
+
+		pid, err := peer.IDB58Decode(spid)
+		if err != nil {
+			return nil, err
+		}
+
+		res[pid] = addrs
+	}
+
+	return res, nil
+}
+
+func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]multiaddr.Multiaddr, len(out.Strings))
+	for i, addr := range out.Strings {
+		ma, err := multiaddr.NewMultiaddr(addr)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = ma
+	}
+	return res, nil
+}
+
+func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]multiaddr.Multiaddr, len(out.Strings))
+	for i, addr := range out.Strings {
+		ma, err := multiaddr.NewMultiaddr(addr)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = ma
+	}
+	return res, nil
+}
+
+func (api *SwarmAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

From c2e0872f6dfba6f6d3f74a4b9eb755053a3f483e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 18:19:50 +0100
Subject: [PATCH 26/64] Implement PubSub Api

This commit was moved from ipfs/go-ipfs-http-client@7abddda1d33595dffb0e0da782b52660147f6855
---
 client/httpapi/api.go      |   2 +-
 client/httpapi/api_test.go |   2 +-
 client/httpapi/pubsub.go   | 129 +++++++++++++++++++++++++++++++++++++
 3 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 client/httpapi/pubsub.go

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index a5b56eff6..3a8e0e003 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -166,5 +166,5 @@ func (api *HttpApi) Swarm() iface.SwarmAPI {
 }
 
 func (api *HttpApi) PubSub() iface.PubSubAPI {
-	return nil
+	return (*PubsubAPI)(api)
 }
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index b822378f7..1acece88c 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -47,7 +47,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
-	startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)}
+	startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)}
 	if err := c.Run(startArgs); err != nil {
 		return nil, err
 	}
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
new file mode 100644
index 000000000..3c5e3c50d
--- /dev/null
+++ b/client/httpapi/pubsub.go
@@ -0,0 +1,129 @@
+package httpapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coreapi/interface"
+	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/libp2p/go-libp2p-peer"
+)
+
+type PubsubAPI HttpApi
+
+func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	return out.Strings, nil
+}
+
+func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) {
+	options, err := caopts.PubSubPeersOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]peer.ID, len(out.Strings))
+	for i, sid := range out.Strings {
+		id, err := peer.IDB58Decode(sid)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = id
+	}
+	return res, nil
+}
+
+func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error {
+	return api.core().request("pubsub/pub", topic).
+		FileBody(bytes.NewReader(message)).
+		Exec(ctx, nil)
+}
+
+type pubsubSub struct {
+	io.Closer
+	dec *json.Decoder
+}
+
+type pubsubMessage struct {
+	JFrom     []byte   `json:"from,omitempty"`
+	JData     []byte   `json:"data,omitempty"`
+	JSeqno    []byte   `json:"seqno,omitempty"`
+	JTopicIDs []string `json:"topicIDs,omitempty"`
+}
+
+func (msg *pubsubMessage) valid() error {
+	_, err := peer.IDFromBytes(msg.JFrom)
+	return err
+}
+
+func (msg *pubsubMessage) From() peer.ID {
+	id, _ := peer.IDFromBytes(msg.JFrom)
+	return id
+}
+
+func (msg *pubsubMessage) Data() []byte {
+	return msg.JData
+}
+
+func (msg *pubsubMessage) Seq() []byte {
+	return msg.JSeqno
+}
+
+func (msg *pubsubMessage) Topics() []string {
+	return msg.JTopicIDs
+}
+
+func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) {
+	// TODO: handle ctx
+
+	var msg pubsubMessage
+	if err := s.dec.Decode(&msg); err != nil {
+		return nil, err
+	}
+	return &msg, msg.valid()
+}
+
+func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) {
+	options, err := caopts.PubSubSubscribeOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := api.core().request("pubsub/sub", topic).
+		Option("discover", options.Discover).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	return &pubsubSub{
+		Closer: resp,
+		dec: json.NewDecoder(resp.Output),
+	}, nil
+}
+
+func (api *PubsubAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}
+

From 9b24cf0aafa94a7b4cb4401b6fbb9c959bde09b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 15 Jan 2019 18:49:11 +0100
Subject: [PATCH 27/64] Use cids in DHT calls

This commit was moved from ipfs/go-ipfs-http-client@163b25f8b88eb98ddae50978440e3940070fabc0
---
 client/httpapi/dht.go    | 20 ++++++++++++++++++--
 client/httpapi/pubsub.go |  3 +--
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go
index 17d3b0500..8fedabbf8 100644
--- a/client/httpapi/dht.go
+++ b/client/httpapi/dht.go
@@ -42,7 +42,13 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop
 	if err != nil {
 		return nil, err
 	}
-	resp, err := api.core().request("dht/findprovs", p.String()).
+
+	rp, err := api.core().ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := api.core().request("dht/findprovs", rp.Cid().String()).
 		Option("num-providers", options.NumProviders).
 		Send(ctx)
 	if err != nil {
@@ -60,6 +66,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop
 
 		for {
 			var out struct {
+				Extra     string
 				Type      notif.QueryEventType
 				Responses []peerstore.PeerInfo
 			}
@@ -67,6 +74,10 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop
 			if err := dec.Decode(&out); err != nil {
 				return // todo: handle this somehow
 			}
+			if out.Type == notif.QueryError {
+				return // usually a 'not found' error
+				// todo: handle other errors
+			}
 			if out.Type == notif.Provider {
 				for _, pi := range out.Responses {
 					select {
@@ -88,7 +99,12 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht
 		return err
 	}
 
-	return api.core().request("dht/provide", p.String()).
+	rp, err := api.core().ResolvePath(ctx, p)
+	if err != nil {
+		return err
+	}
+
+	return api.core().request("dht/provide", rp.Cid().String()).
 		Option("recursive", options.Recursive).
 		Exec(ctx, nil)
 }
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index 3c5e3c50d..fb9bb7460 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -119,11 +119,10 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 
 	return &pubsubSub{
 		Closer: resp,
-		dec: json.NewDecoder(resp.Output),
+		dec:    json.NewDecoder(resp.Output),
 	}, nil
 }
 
 func (api *PubsubAPI) core() *HttpApi {
 	return (*HttpApi)(api)
 }
-

From b7e258cc1015e518198c50b02fcc6ebdd8767375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 21 Jan 2019 21:31:09 +0100
Subject: [PATCH 28/64] Reimplement DAG as DAGService

This commit was moved from ipfs/go-ipfs-http-client@f34a5f6d2569bb21f022d3231e811b3db6262ca8
---
 client/httpapi/api.go     |   6 +-
 client/httpapi/apifile.go |   5 ++
 client/httpapi/dag.go     | 115 ++++++++++++++++++++++++--------------
 3 files changed, 83 insertions(+), 43 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 3a8e0e003..7d4fbd0e0 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	"github.com/ipfs/go-ipld-format"
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
@@ -137,8 +139,8 @@ func (api *HttpApi) Block() iface.BlockAPI {
 	return (*BlockAPI)(api)
 }
 
-func (api *HttpApi) Dag() iface.DagAPI {
-	return (*DagAPI)(api)
+func (api *HttpApi) Dag() format.DAGService {
+	return (*HttpDagServ)(api)
 }
 
 func (api *HttpApi) Name() iface.NameAPI {
diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index d9da23975..99ae72059 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -125,6 +125,11 @@ func (it *apiIter) Name() string {
 }
 
 func (it *apiIter) Next() bool {
+	if it.ctx.Err() != nil {
+		it.err = it.ctx.Err()
+		return false
+	}
+
 	var out lsOutput
 	if err := it.dec.Decode(&out); err != nil {
 		if err != io.EOF {
diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index 20c233196..de2934c2f 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -1,71 +1,104 @@
 package httpapi
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"math"
+	"io/ioutil"
+	"sync"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
+	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipld-format"
-	mh "github.com/multiformats/go-multihash"
 )
 
-type DagAPI HttpApi
+type HttpDagServ HttpApi
 
-func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) {
-	options, err := caopts.DagPutOptions(opts...)
+func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
+	r, err := api.core().Block().Get(ctx, iface.IpldPath(c))
 	if err != nil {
 		return nil, err
 	}
 
-	codec, ok := cid.CodecToStr[options.Codec]
-	if !ok {
-		return nil, fmt.Errorf("unknowm codec %d", options.MhType)
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
 	}
 
-	if options.MhLength != -1 {
-		return nil, fmt.Errorf("setting hash len is not supported yet")
+	blk, err := blocks.NewBlockWithCid(data, c)
+	if err != nil {
+		return nil, err
 	}
 
-	var out struct {
-		Cid cid.Cid
-	}
-	req := api.core().request("dag/put").
-		Option("format", codec).
-		Option("input-enc", options.InputEnc)
+	return format.DefaultBlockDecoder.Decode(blk)
+}
 
-	if options.MhType != math.MaxUint64 {
-		mht, ok := mh.Codes[options.MhType]
-		if !ok {
-			return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption {
+	out := make(chan *format.NodeOption)
+	wg := sync.WaitGroup{}
+	wg.Add(len(cids))
+
+	for _, c := range cids {
+		// TODO: Consider limiting concurrency of this somehow
+		go func() {
+			defer wg.Done()
+			n, err := api.Get(ctx, c)
+
+			select {
+			case out <- &format.NodeOption{Node: n, Err: err}:
+			case <-ctx.Done():
+			}
+		}()
+	}
+	return out
+}
+
+func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
+	c := nd.Cid()
+	prefix := c.Prefix()
+	format := cid.CodecToStr[prefix.Codec]
+	if prefix.Version == 0 {
+		format = "v0"
+	}
+
+	stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()),
+		options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format))
+	if err != nil {
+		return err
+	}
+	if !stat.Path().Cid().Equals(c) {
+		return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String())
+	}
+	return nil
+}
+
+func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error {
+	for _, nd := range nds {
+		// TODO: optimize
+		if err := api.Add(ctx, nd); err != nil {
+			return err
 		}
-		req.Option("hash", mht)
 	}
+	return nil
+}
 
-	err = req.FileBody(src).Exec(ctx, &out)
-	if err != nil {
-		return nil, err
+func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error {
+	return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm?
+}
+
+func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error {
+	for _, c := range cids {
+		// TODO: optimize
+		if err := api.Remove(ctx, c); err != nil {
+			return err
+		}
 	}
-
-	return iface.IpldPath(out.Cid), nil
+	return nil
 }
 
-func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) {
-	panic("implement me")
-}
-
-func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) {
-	panic("implement me")
-}
-
-func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch {
-	panic("implement me")
-}
-
-func (api *DagAPI) core() *HttpApi {
+func (api *HttpDagServ) core() *HttpApi {
 	return (*HttpApi)(api)
 }

From c3f2970f8a7302344154e6d7fe74485f0dd272c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:42:32 +0100
Subject: [PATCH 29/64] block: Pin option

This commit was moved from ipfs/go-ipfs-http-client@def66919dfbee162595852334468719f57158121
---
 client/httpapi/block.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index f8334a109..24df9629e 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -53,6 +53,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP
 		Option("mhtype", mht).
 		Option("mhlen", options.MhLength).
 		Option("format", options.Codec).
+		Option("pin", options.Pin).
 		FileBody(r)
 
 	var out blockStat

From af2edd12eb44f477ef2e0dc22ae31a75df645b9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:43:00 +0100
Subject: [PATCH 30/64] tests: enable filestore

This commit was moved from ipfs/go-ipfs-http-client@904e8eeeb1b677d458c819122d91264040120d3b
---
 client/httpapi/api_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 1acece88c..bf1d6b9b0 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -47,6 +47,11 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
+	filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"}
+	if err := c.Run(filestoreArgs); err != nil {
+		return nil, err
+	}
+
 	startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)}
 	if err := c.Run(startArgs); err != nil {
 		return nil, err

From 75cf2be1026a7f206e61d164342dae4807d36bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:43:26 +0100
Subject: [PATCH 31/64] apifile: Implement Seek

This commit was moved from ipfs/go-ipfs-http-client@69cc3e8106552605ecfdfa5c6f352996dc6f9497
---
 client/httpapi/apifile.go | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index 99ae72059..ec916d8e2 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -6,13 +6,15 @@ import (
 	"fmt"
 	"github.com/ipfs/go-cid"
 	"io"
+	"io/ioutil"
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 
 	"github.com/ipfs/go-ipfs-files"
-	unixfspb "github.com/ipfs/go-unixfs/pb"
 )
 
+const forwardSeekLimit = 1 << 14 //16k
+
 func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) {
 	if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS
 		var err error
@@ -80,7 +82,24 @@ func (f *apiFile) Read(p []byte) (int, error) {
 }
 
 func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
-	panic("implement me") //TODO
+	switch whence {
+	case io.SeekEnd:
+		offset = f.size + offset
+	case io.SeekCurrent:
+		offset = f.at + offset
+	}
+	if f.at == offset { //noop
+		return offset, nil
+	}
+
+	if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip
+		r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at)
+
+		f.at += r
+		return f.at, err
+	}
+	f.at = offset
+	return f.at, f.reset()
 }
 
 func (f *apiFile) Close() error {
@@ -156,17 +175,17 @@ func (it *apiIter) Next() bool {
 	}
 
 	switch it.cur.Type {
-	case unixfspb.Data_HAMTShard:
+	case iface.THAMTShard:
 		fallthrough
-	case unixfspb.Data_Metadata:
+	case iface.TMetadata:
 		fallthrough
-	case unixfspb.Data_Directory:
+	case iface.TDirectory:
 		it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size))
 		if err != nil {
 			it.err = err
 			return false
 		}
-	case unixfspb.Data_File:
+	case iface.TFile:
 		it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size))
 		if err != nil {
 			it.err = err

From bb83ccb15feb4854c7bdb4821fbcf90b89f36163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:44:48 +0100
Subject: [PATCH 32/64] response: read trailing error headers

This commit was moved from ipfs/go-ipfs-http-client@83dfd84ba8bf2341478bb98d437ae1f14fe29e18
---
 client/httpapi/response.go | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index f6e7f3ab7..745c9a2a9 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -2,6 +2,7 @@ package httpapi
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,6 +14,24 @@ import (
 	files "github.com/ipfs/go-ipfs-files"
 )
 
+type trailerReader struct {
+	resp *http.Response
+}
+
+func (r *trailerReader) Read(b []byte) (int, error) {
+	n, err := r.resp.Body.Read(b)
+	if err != nil {
+		if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" {
+			err = errors.New(e)
+		}
+	}
+	return n, err
+}
+
+func (r *trailerReader) Close() error {
+	return r.resp.Body.Close()
+}
+
 type Response struct {
 	Output io.ReadCloser
 	Error  *Error
@@ -56,9 +75,6 @@ type Error struct {
 
 func (e *Error) Error() string {
 	var out string
-	if e.Command != "" {
-		out = e.Command + ": "
-	}
 	if e.Code != 0 {
 		out = fmt.Sprintf("%s%d: ", out, e.Code)
 	}
@@ -93,7 +109,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 
 	nresp := new(Response)
 
-	nresp.Output = resp.Body
+	nresp.Output = &trailerReader{resp}
 	if resp.StatusCode >= http.StatusBadRequest {
 		e := &Error{
 			Command: r.Command,

From 88139ddc50e111e39721990bb86bb1c018c488b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:45:24 +0100
Subject: [PATCH 33/64] dag: Interface updates

This commit was moved from ipfs/go-ipfs-http-client@7bea2efb45cba072b4e24c20b46a4372be3a716d
---
 client/httpapi/api.go |  3 +--
 client/httpapi/dag.go | 38 +++++++++++++++++++++++++++++++++-----
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 7d4fbd0e0..2c7a97c99 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -11,7 +11,6 @@ import (
 	"github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
-	"github.com/ipfs/go-ipld-format"
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
@@ -139,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI {
 	return (*BlockAPI)(api)
 }
 
-func (api *HttpApi) Dag() format.DAGService {
+func (api *HttpApi) Dag() iface.APIDagService {
 	return (*HttpDagServ)(api)
 }
 
diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index de2934c2f..eacf631b9 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -15,7 +15,9 @@ import (
 	"github.com/ipfs/go-ipld-format"
 )
 
-type HttpDagServ HttpApi
+type httpNodeAdder HttpApi
+type HttpDagServ httpNodeAdder
+type pinningHttpNodeAdder httpNodeAdder
 
 func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
 	r, err := api.core().Block().Get(ctx, iface.IpldPath(c))
@@ -56,7 +58,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for
 	return out
 }
 
-func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
+func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error {
 	c := nd.Cid()
 	prefix := c.Prefix()
 	format := cid.CodecToStr[prefix.Codec]
@@ -65,7 +67,9 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
 	}
 
 	stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()),
-		options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format))
+		options.Block.Hash(prefix.MhType, prefix.MhLength),
+		options.Block.Format(format),
+		options.Block.Pin(pin))
 	if err != nil {
 		return err
 	}
@@ -75,16 +79,36 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
 	return nil
 }
 
-func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error {
+func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error {
 	for _, nd := range nds {
 		// TODO: optimize
-		if err := api.Add(ctx, nd); err != nil {
+		if err := api.add(ctx, nd, pin); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
+func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error {
+	return (*httpNodeAdder)(api).addMany(ctx, nds, false)
+}
+
+func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
+	return (*httpNodeAdder)(api).add(ctx, nd, false)
+}
+
+func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error {
+	return (*httpNodeAdder)(api).add(ctx, nd, true)
+}
+
+func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error {
+	return (*httpNodeAdder)(api).addMany(ctx, nds, true)
+}
+
+func (api *HttpDagServ) Pinning() format.NodeAdder {
+	return (*pinningHttpNodeAdder)(api)
+}
+
 func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error {
 	return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm?
 }
@@ -99,6 +123,10 @@ func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error {
 	return nil
 }
 
+func (api *httpNodeAdder) core() *HttpApi {
+	return (*HttpApi)(api)
+}
+
 func (api *HttpDagServ) core() *HttpApi {
 	return (*HttpApi)(api)
 }

From 93bfcf91cf979b267758c04aa14c6435a687fc36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:46:00 +0100
Subject: [PATCH 34/64] unixfs: updated ls

This commit was moved from ipfs/go-ipfs-http-client@bb8d9d1a60e4161c4cf3a33c468746050f267a12
---
 client/httpapi/unixfs.go | 94 ++++++++++++++++++++++++++++++++--------
 1 file changed, 75 insertions(+), 19 deletions(-)

diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 7e63d241f..67d05309b 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -13,7 +13,6 @@ import (
 
 	"github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipld-format"
-	unixfspb "github.com/ipfs/go-unixfs/pb"
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -129,7 +128,7 @@ loop:
 type lsLink struct {
 	Name, Hash string
 	Size       uint64
-	Type       unixfspb.Data_DataType
+	Type       iface.FileType
 }
 
 type lsObject struct {
@@ -141,30 +140,87 @@ type lsOutput struct {
 	Objects []lsObject
 }
 
-func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) {
-	var out lsOutput
-	err := api.core().request("ls", p.String()).Exec(ctx, &out)
+func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) {
+	options, err := caopts.UnixfsLsOptions(opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(out.Objects) != 1 {
-		return nil, errors.New("unexpected objects len")
+	resp, err := api.core().request("ls", p.String()).
+		Option("resolve-type", options.ResolveChildren).
+		Option("size", options.ResolveChildren).
+		Option("stream", true).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
 	}
 
-	links := make([]*format.Link, len(out.Objects[0].Links))
-	for i, l := range out.Objects[0].Links {
-		c, err := cid.Parse(l.Hash)
-		if err != nil {
-			return nil, err
+	dec := json.NewDecoder(resp.Output)
+	out := make(chan iface.LsLink)
+
+	go func() {
+		defer resp.Close()
+		defer close(out)
+
+		for {
+			var link lsOutput
+			if err := dec.Decode(&link); err != nil {
+				if err == io.EOF {
+					return
+				}
+				select {
+				case out <- iface.LsLink{Err: err}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			if len(link.Objects) != 1 {
+				select {
+				case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			if len(link.Objects[0].Links) != 1 {
+				select {
+				case out <- iface.LsLink{Err: errors.New("unexpected Links len")}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			l0 := link.Objects[0].Links[0]
+
+			c, err := cid.Decode(l0.Hash)
+			if err != nil {
+				select {
+				case out <- iface.LsLink{Err: err}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			select {
+			case out <- iface.LsLink{
+				Link: &format.Link{
+					Cid: c,
+					Name: l0.Name,
+					Size: l0.Size,
+				},
+				Size: l0.Size,
+				Type: l0.Type,
+			}:
+			case <-ctx.Done():
+			}
 		}
-		links[i] = &format.Link{
-			Name: l.Name,
-			Size: l.Size,
-			Cid:  c,
-		}
-	}
-	return links, nil
+	}()
+
+	return out, nil
 }
 
 func (api *UnixfsAPI) core() *HttpApi {

From 1bc854bf2169f82d2275232e123adae721b8dc0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 4 Feb 2019 19:53:02 +0100
Subject: [PATCH 35/64] pin: handle Rm options

This commit was moved from ipfs/go-ipfs-http-client@e85e856ea2841cd3e0cf4cb7b87441539af42bbe
---
 client/httpapi/pin.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index 1624b7867..237b32329 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -68,8 +68,15 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.
 	return pins, nil
 }
 
-func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error {
-	return api.core().request("pin/rm", p.String()).Exec(ctx, nil)
+func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error {
+	options, err := caopts.PinRmOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().request("pin/rm", p.String()).
+		Option("recursive", options.Recursive).
+		Exec(ctx, nil)
 }
 
 func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error {

From c543354b1772323cb2cfbf41c06522297bf27667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 6 Feb 2019 22:36:30 +0100
Subject: [PATCH 36/64] Check for redirects

This commit was moved from ipfs/go-ipfs-http-client@19c65db4f0fd1549fc6f224efd868d4f00da997d
---
 client/httpapi/api.go            | 18 +++++++++++++-----
 client/httpapi/requestbuilder.go |  2 +-
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 2c7a97c99..da0836d4a 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -2,6 +2,7 @@ package httpapi
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	gohttp "net/http"
 	"os"
@@ -27,7 +28,7 @@ var ErrNotImplemented = errors.New("not implemented")
 
 type HttpApi struct {
 	url     string
-	httpcli *gohttp.Client
+	httpcli gohttp.Client
 
 	applyGlobal func(*RequestBuilder)
 }
@@ -50,8 +51,8 @@ func NewPathApi(p string) iface.CoreAPI {
 	return NewApi(a)
 }
 
-func ApiAddr(p string) ma.Multiaddr {
-	baseDir, err := homedir.Expand(p)
+func ApiAddr(ipfspath string) ma.Multiaddr {
+	baseDir, err := homedir.Expand(ipfspath)
 	if err != nil {
 		return nil
 	}
@@ -99,11 +100,18 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
 		}
 	}
 
-	return &HttpApi{
+	api := &HttpApi{
 		url:         url,
-		httpcli:     c,
+		httpcli:     *c,
 		applyGlobal: func(*RequestBuilder) {},
 	}
+
+	// We don't support redirects.
+	api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error {
+		return fmt.Errorf("unexpected redirect")
+	}
+
+	return api
 }
 
 func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) {
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index 831e6d71c..628ad03cd 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -92,7 +92,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) {
 	req.Opts = r.opts
 	req.Headers = r.headers
 	req.Body = r.body
-	return req.Send(r.shell.httpcli)
+	return req.Send(&r.shell.httpcli)
 }
 
 // Exec sends the request a request and decodes the response.

From ef5bf40df1d0d790aeb213147c53412145894bf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 11 Feb 2019 18:40:38 +0100
Subject: [PATCH 37/64] Update imports to use extracted interface

This commit was moved from ipfs/go-ipfs-http-client@fad467bc437a1282ed3813d828c6f8ec5b7b2d7a
---
 client/httpapi/api.go      |  5 ++---
 client/httpapi/api_test.go |  7 +++----
 client/httpapi/apifile.go  |  9 ++++-----
 client/httpapi/block.go    |  4 ++--
 client/httpapi/dag.go      |  5 ++---
 client/httpapi/dht.go      |  4 ++--
 client/httpapi/ipldnode.go |  3 +--
 client/httpapi/key.go      |  5 ++---
 client/httpapi/name.go     |  6 +++---
 client/httpapi/object.go   |  5 ++---
 client/httpapi/path.go     |  3 +--
 client/httpapi/pin.go      |  8 ++++----
 client/httpapi/pubsub.go   |  5 ++---
 client/httpapi/swarm.go    |  5 ++---
 client/httpapi/unixfs.go   | 11 +++++------
 15 files changed, 37 insertions(+), 48 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index da0836d4a..698e36524 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -9,9 +9,8 @@ import (
 	"path"
 	"strings"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	homedir "github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index bf1d6b9b0..5621ab87f 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -10,10 +10,9 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/tests"
-
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
+	"github.com/ipfs/interface-go-ipfs-core/tests"
 	local "github.com/ipfs/iptb-plugins/local"
 	"github.com/ipfs/iptb/cli"
 	"github.com/ipfs/iptb/testbed"
diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index ec916d8e2..1e5f61a9a 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -4,13 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/ipfs/go-cid"
 	"io"
 	"io/ioutil"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs-files"
+	"github.com/ipfs/interface-go-ipfs-core"
 )
 
 const forwardSeekLimit = 1 << 14 //16k
@@ -92,8 +91,8 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
 		return offset, nil
 	}
 
-	if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip
-		r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at)
+	if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip
+		r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at)
 
 		f.at += r
 		return f.at, err
diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index 24df9629e..45c73472c 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -8,8 +8,8 @@ import (
 	"io"
 
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	mh "github.com/multiformats/go-multihash"
 )
 
diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index eacf631b9..d0059698a 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -7,12 +7,11 @@ import (
 	"io/ioutil"
 	"sync"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
 	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/interface-go-ipfs-core/options"
 )
 
 type httpNodeAdder HttpApi
diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go
index 8fedabbf8..dc7dd6bea 100644
--- a/client/httpapi/dht.go
+++ b/client/httpapi/dht.go
@@ -3,9 +3,9 @@ package httpapi
 import (
 	"context"
 	"encoding/json"
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/libp2p/go-libp2p-peer"
 	"github.com/libp2p/go-libp2p-peerstore"
 	notif "github.com/libp2p/go-libp2p-routing/notifications"
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
index 2294a95b4..43fa5d50d 100644
--- a/client/httpapi/ipldnode.go
+++ b/client/httpapi/ipldnode.go
@@ -6,11 +6,10 @@ import (
 	"io/ioutil"
 	"strconv"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	ipfspath "github.com/ipfs/go-path"
+	"github.com/ipfs/interface-go-ipfs-core"
 )
 
 type ipldNode struct {
diff --git a/client/httpapi/key.go b/client/httpapi/key.go
index 5087b6d99..dc2adf8b7 100644
--- a/client/httpapi/key.go
+++ b/client/httpapi/key.go
@@ -4,9 +4,8 @@ import (
 	"context"
 	"errors"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/libp2p/go-libp2p-peer"
 )
 
diff --git a/client/httpapi/name.go b/client/httpapi/name.go
index fbc440b96..d760e6188 100644
--- a/client/httpapi/name.go
+++ b/client/httpapi/name.go
@@ -3,10 +3,10 @@ package httpapi
 import (
 	"context"
 	"fmt"
-	"github.com/ipfs/go-ipfs/namesys/opts"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
+	"github.com/ipfs/interface-go-ipfs-core/options/namesys"
 )
 
 type NameAPI HttpApi
diff --git a/client/httpapi/object.go b/client/httpapi/object.go
index 4bc787991..3b648d82b 100644
--- a/client/httpapi/object.go
+++ b/client/httpapi/object.go
@@ -6,12 +6,11 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-merkledag"
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 )
 
 type ObjectAPI HttpApi
diff --git a/client/httpapi/path.go b/client/httpapi/path.go
index 5701326fc..68dc2633d 100644
--- a/client/httpapi/path.go
+++ b/client/httpapi/path.go
@@ -3,11 +3,10 @@ package httpapi
 import (
 	"context"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-
 	cid "github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	ipfspath "github.com/ipfs/go-path"
+	"github.com/ipfs/interface-go-ipfs-core"
 )
 
 func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) {
diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index 237b32329..4c4e5713c 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -3,11 +3,11 @@ package httpapi
 import (
 	"context"
 	"encoding/json"
-	"github.com/ipfs/go-cid"
-	"github.com/pkg/errors"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
+	"github.com/pkg/errors"
 )
 
 type PinAPI HttpApi
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index fb9bb7460..334509780 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -6,9 +6,8 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/libp2p/go-libp2p-peer"
 )
 
diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
index cf0ee2a9b..d179b6540 100644
--- a/client/httpapi/swarm.go
+++ b/client/httpapi/swarm.go
@@ -2,14 +2,13 @@ package httpapi
 
 import (
 	"context"
-	"github.com/libp2p/go-libp2p-protocol"
 	"time"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-
+	"github.com/ipfs/interface-go-ipfs-core"
 	inet "github.com/libp2p/go-libp2p-net"
 	"github.com/libp2p/go-libp2p-peer"
 	"github.com/libp2p/go-libp2p-peerstore"
+	"github.com/libp2p/go-libp2p-protocol"
 	"github.com/multiformats/go-multiaddr"
 )
 
diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go
index 67d05309b..1f340b657 100644
--- a/client/httpapi/unixfs.go
+++ b/client/httpapi/unixfs.go
@@ -3,16 +3,15 @@ package httpapi
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
-	"github.com/ipfs/go-cid"
-	"github.com/pkg/errors"
 	"io"
 
-	"github.com/ipfs/go-ipfs/core/coreapi/interface"
-	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
-
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/interface-go-ipfs-core"
+	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -208,7 +207,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf
 			select {
 			case out <- iface.LsLink{
 				Link: &format.Link{
-					Cid: c,
+					Cid:  c,
 					Name: l0.Name,
 					Size: l0.Size,
 				},

From 0813d808b54341414441a56e5cd3a7722a516a14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 12 Feb 2019 12:29:31 +0100
Subject: [PATCH 38/64] Fix govet warning in Dag

This commit was moved from ipfs/go-ipfs-http-client@62552b33959401c7a9d9463dffba433e522b2f15
---
 client/httpapi/dag.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index d0059698a..3f54ced34 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -44,7 +44,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for
 
 	for _, c := range cids {
 		// TODO: Consider limiting concurrency of this somehow
-		go func() {
+		go func(c cid.Cid) {
 			defer wg.Done()
 			n, err := api.Get(ctx, c)
 
@@ -52,7 +52,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for
 			case out <- &format.NodeOption{Node: n, Err: err}:
 			case <-ctx.Done():
 			}
-		}()
+		}(c)
 	}
 	return out
 }

From abc30e384fafe157490df0dae80a22ddeb6255a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 12 Feb 2019 12:48:04 +0100
Subject: [PATCH 39/64] Register iptb plugin once

This commit was moved from ipfs/go-ipfs-http-client@d6c8cbd5e643d2131477120a76159074d1a07cd3
---
 client/httpapi/api_test.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 5621ab87f..fa38866e1 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -19,9 +19,7 @@ import (
 	"github.com/ipfs/iptb/testbed/interfaces"
 )
 
-type NodeProvider struct{}
-
-func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
+func init() {
 	_, err := testbed.RegisterPlugin(testbed.IptbPlugin{
 		From:        "<builtin>",
 		NewNode:     local.NewNode,
@@ -31,8 +29,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		BuiltIn:     true,
 	}, false)
 	if err != nil {
-		return nil, err
+		panic(err)
 	}
+}
+
+type NodeProvider struct{}
+
+func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
 
 	dir, err := ioutil.TempDir("", "httpapi-tb-")
 	if err != nil {

From 9a6ee6f5e153b96afda0f4ca1ee893cf28aa4f57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 17:30:04 +0100
Subject: [PATCH 40/64] Improve test node spawning

This commit was moved from ipfs/go-ipfs-http-client@6bb2a287a6f5090792e07f3e3a74871c459b395f
---
 client/httpapi/api_test.go | 198 ++++++++++++++++++++++++-------------
 1 file changed, 131 insertions(+), 67 deletions(-)

diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index fa38866e1..af2180384 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -2,23 +2,23 @@ package httpapi
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	gohttp "net/http"
 	"os"
-	"path"
 	"strconv"
+	"sync"
 	"testing"
 
 	"github.com/ipfs/interface-go-ipfs-core"
-	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/tests"
 	local "github.com/ipfs/iptb-plugins/local"
-	"github.com/ipfs/iptb/cli"
 	"github.com/ipfs/iptb/testbed"
 	"github.com/ipfs/iptb/testbed/interfaces"
+	ma "github.com/multiformats/go-multiaddr"
 )
 
+const parallelSpeculativeNodes = 15 // 15 seems to work best
+
 func init() {
 	_, err := testbed.RegisterPlugin(testbed.IptbPlugin{
 		From:        "<builtin>",
@@ -33,99 +33,163 @@ func init() {
 	}
 }
 
-type NodeProvider struct{}
+type NodeProvider struct {
+	simple <-chan func(context.Context) ([]iface.CoreAPI, error)
+}
 
-func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
+func newNodeProvider(ctx context.Context) *NodeProvider {
+	simpleNodes := make(chan func(context.Context) ([]iface.CoreAPI, error), parallelSpeculativeNodes)
+
+	np := &NodeProvider{
+		simple: simpleNodes,
+	}
+
+	// start basic nodes speculatively in parallel
+	for i := 0; i < parallelSpeculativeNodes; i++ {
+		go func() {
+			for {
+				ctx, cancel := context.WithCancel(ctx)
+
+				snd, err := np.makeAPISwarm(ctx, false, 1)
+
+				res := func(ctx context.Context) ([]iface.CoreAPI, error) {
+					if err != nil {
+						return nil, err
+					}
+
+					go func() {
+						<-ctx.Done()
+						cancel()
+					}()
+
+					return snd, nil
+				}
+
+				select {
+				case simpleNodes <- res:
+				case <-ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	return np
+}
+
+func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
+	if !fullIdentity && n == 1 {
+		return (<-np.simple)(ctx)
+	}
+	return np.makeAPISwarm(ctx, fullIdentity, n)
+}
+
+func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) {
 
 	dir, err := ioutil.TempDir("", "httpapi-tb-")
 	if err != nil {
 		return nil, err
 	}
 
-	c := cli.NewCli() //TODO: is there a better way?
+	tb := testbed.NewTestbed(dir)
 
-	initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)}
-	if err := c.Run(initArgs); err != nil {
+	specs, err := testbed.BuildSpecs(tb.Dir(), n, "localipfs", nil)
+	if err != nil {
 		return nil, err
 	}
 
-	filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"}
-	if err := c.Run(filestoreArgs); err != nil {
+	if err := testbed.WriteNodeSpecs(tb.Dir(), specs); err != nil {
 		return nil, err
 	}
 
-	startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)}
-	if err := c.Run(startArgs); err != nil {
+	nodes, err := tb.Nodes()
+	if err != nil {
 		return nil, err
 	}
 
-	if n > 1 {
-		connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"}
-		if err := c.Run(connectArgs); err != nil {
-			return nil, err
-		}
+	apis := make([]iface.CoreAPI, n)
+
+	wg := sync.WaitGroup{}
+	zero := sync.WaitGroup{}
+
+	wg.Add(len(nodes))
+	zero.Add(1)
+
+	for i, nd := range nodes {
+		go func(i int, nd testbedi.Core) {
+			defer wg.Done()
+
+			if _, err := nd.Init(ctx, "--empty-repo"); err != nil {
+				panic(err)
+			}
+
+			if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil {
+				panic(err)
+			}
+
+			if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil {
+				panic(err)
+			}
+
+			if i > 0 {
+				zero.Wait()
+				if err := nd.Connect(ctx, nodes[0]); err != nil {
+					panic(err)
+				}
+			} else {
+				zero.Done()
+			}
+
+			addr, err := nd.APIAddr()
+			if err != nil {
+				panic(err)
+			}
+
+			maddr, err := ma.NewMultiaddr(addr)
+			if err != nil {
+				panic(err)
+			}
+
+			c := &gohttp.Client{
+				Transport: &gohttp.Transport{
+					Proxy:              gohttp.ProxyFromEnvironment,
+					DisableKeepAlives:  true,
+					DisableCompression: true,
+				},
+			}
+			apis[i] = NewApiWithClient(maddr, c)
+
+			// empty node is pinned even with --empty-repo, we don't want that
+			emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+			if err != nil {
+				panic(err)
+			}
+			if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil {
+				panic(err)
+			}
+		}(i, nd)
 	}
 
+	wg.Wait()
+
 	go func() {
 		<-ctx.Done()
 
 		defer os.Remove(dir)
 
 		defer func() {
-			_ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"})
+			for _, nd := range nodes {
+				_ = nd.Stop(context.Background())
+			}
 		}()
 	}()
 
-	apis := make([]iface.CoreAPI, n)
-
-	for i := range apis {
-		tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default"))
-
-		node, err := tb.Node(i)
-		if err != nil {
-			return nil, err
-		}
-
-		attrNode, ok := node.(testbedi.Attribute)
-		if !ok {
-			return nil, fmt.Errorf("node does not implement attributes")
-		}
-
-		pth, err := attrNode.Attr("path")
-		if err != nil {
-			return nil, err
-		}
-
-		a := ApiAddr(pth)
-		if a == nil {
-			return nil, fmt.Errorf("nil addr for node")
-		}
-		c := &gohttp.Client{
-			Transport: &gohttp.Transport{
-				Proxy:              gohttp.ProxyFromEnvironment,
-				DisableKeepAlives:  true,
-				DisableCompression: true,
-			},
-		}
-		apis[i] = NewApiWithClient(a, c)
-
-		// node cleanup
-		// TODO: pass --empty-repo somehow (how?)
-		pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive())
-		if err != nil {
-			return nil, err
-		}
-		for _, pin := range pins { //TODO: parallel
-			if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil {
-				return nil, err
-			}
-		}
-
-	}
-
 	return apis, nil
 }
 
 func TestHttpApi(t *testing.T) {
-	tests.TestApi(&NodeProvider{})(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	tests.TestApi(newNodeProvider(ctx))(t)
 }

From f7dd0c690960ea79a7aad7f40b11362512d173bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 18:12:02 +0100
Subject: [PATCH 41/64] Improve apifile error messages

This commit was moved from ipfs/go-ipfs-http-client@3393b8379021695126da83afdf3bf54eaff828f4
---
 client/httpapi/apifile.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index 1e5f61a9a..864760684 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -157,12 +157,12 @@ func (it *apiIter) Next() bool {
 	}
 
 	if len(out.Objects) != 1 {
-		it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects))
+		it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects))
 		return false
 	}
 
 	if len(out.Objects[0].Links) != 1 {
-		it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links))
+		it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links))
 		return false
 	}
 

From e1b14d78c72edf0a782e362afb0e818e032c31cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 18:45:19 +0100
Subject: [PATCH 42/64] Implement missing methods

This commit was moved from ipfs/go-ipfs-http-client@b76413dfe55fd33d39a680458282d85565cc1c69
---
 client/httpapi/api.go  |  3 --
 client/httpapi/name.go | 63 +++++++++++++++++++++++++++++++++++++++---
 client/httpapi/path.go |  9 ++++--
 3 files changed, 66 insertions(+), 9 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 698e36524..e106d8d87 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -1,7 +1,6 @@
 package httpapi
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	gohttp "net/http"
@@ -23,8 +22,6 @@ const (
 	EnvDir          = "IPFS_PATH"
 )
 
-var ErrNotImplemented = errors.New("not implemented")
-
 type HttpApi struct {
 	url     string
 	httpcli gohttp.Client
diff --git a/client/httpapi/name.go b/client/httpapi/name.go
index d760e6188..58fb59249 100644
--- a/client/httpapi/name.go
+++ b/client/httpapi/name.go
@@ -2,7 +2,9 @@ package httpapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 
 	"github.com/ipfs/interface-go-ipfs-core"
 	caopts "github.com/ipfs/interface-go-ipfs-core/options"
@@ -38,11 +40,11 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na
 	req := api.core().request("name/publish", p.String()).
 		Option("key", options.Key).
 		Option("allow-offline", options.AllowOffline).
-		Option("lifetime", options.ValidTime.String()).
+		Option("lifetime", options.ValidTime).
 		Option("resolve", false)
 
 	if options.TTL != nil {
-		req.Option("ttl", options.TTL.String())
+		req.Option("ttl", options.TTL)
 	}
 
 	var out ipnsEntry
@@ -57,7 +59,60 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na
 }
 
 func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) {
-	return nil, ErrNotImplemented
+	options, err := caopts.NameResolveOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	ropts := nsopts.ProcessOpts(options.ResolveOpts)
+	if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 {
+		return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit)
+	}
+
+	req := api.core().request("name/resolve", name).
+		Option("nocache", !options.Cache).
+		Option("recursive", ropts.Depth != 1).
+		Option("dht-record-count", ropts.DhtRecordCount).
+		Option("dht-timeout", ropts.DhtTimeout).
+		Option("stream", true)
+	resp, err := req.Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	res := make(chan iface.IpnsResult)
+
+	go func() {
+		defer close(res)
+		defer resp.Close()
+
+		dec := json.NewDecoder(resp.Output)
+
+		for {
+			var out struct{ Path string }
+			err := dec.Decode(&out)
+			if err == io.EOF {
+				return
+			}
+			var ires iface.IpnsResult
+			if err == nil {
+				ires.Path, err = iface.ParsePath(out.Path)
+			}
+
+			select {
+			case res <- ires:
+			case <-ctx.Done():
+			}
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	return res, nil
 }
 
 func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) {
@@ -75,7 +130,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam
 		Option("nocache", !options.Cache).
 		Option("recursive", ropts.Depth != 1).
 		Option("dht-record-count", ropts.DhtRecordCount).
-		Option("dht-timeout", ropts.DhtTimeout.String())
+		Option("dht-timeout", ropts.DhtTimeout)
 
 	var out struct{ Path string }
 	if err := req.Exec(ctx, &out); err != nil {
diff --git a/client/httpapi/path.go b/client/httpapi/path.go
index 68dc2633d..8c819121a 100644
--- a/client/httpapi/path.go
+++ b/client/httpapi/path.go
@@ -42,6 +42,11 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res
 	return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil
 }
 
-func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) {
-	return nil, ErrNotImplemented
+func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return api.Dag().Get(ctx, rp.Cid())
 }

From 42273cab06348b19a813dee77f7cbe5ed7cc78d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 19:05:17 +0100
Subject: [PATCH 43/64] Don't use valid() pattern

This commit was moved from ipfs/go-ipfs-http-client@4d07c48f98b0af0b18c6c13f80f04c966fc43bdc
---
 client/httpapi/block.go  | 20 +++++++-------------
 client/httpapi/key.go    | 28 ++++++++++++++++------------
 client/httpapi/name.go   | 14 ++++----------
 client/httpapi/pubsub.go | 12 +++++-------
 4 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/client/httpapi/block.go b/client/httpapi/block.go
index 45c73472c..fd4d9bab9 100644
--- a/client/httpapi/block.go
+++ b/client/httpapi/block.go
@@ -18,24 +18,16 @@ type BlockAPI HttpApi
 type blockStat struct {
 	Key   string
 	BSize int `json:"Size"`
+
+	cid cid.Cid
 }
 
 func (s *blockStat) Size() int {
 	return s.BSize
 }
 
-func (s *blockStat) valid() (iface.ResolvedPath, error) {
-	c, err := cid.Parse(s.Key)
-	if err != nil {
-		return nil, err
-	}
-
-	return iface.IpldPath(c), nil
-}
-
 func (s *blockStat) Path() iface.ResolvedPath {
-	p, _ := s.valid()
-	return p
+	return iface.IpldPath(s.cid)
 }
 
 func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) {
@@ -60,7 +52,8 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP
 	if err := req.Exec(ctx, &out); err != nil {
 		return nil, err
 	}
-	if _, err := out.valid(); err != nil {
+	out.cid, err = cid.Parse(out.Key)
+	if err != nil {
 		return nil, err
 	}
 
@@ -118,7 +111,8 @@ func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, e
 	if err != nil {
 		return nil, err
 	}
-	if _, err := out.valid(); err != nil {
+	out.cid, err = cid.Parse(out.Key)
+	if err != nil {
 		return nil, err
 	}
 
diff --git a/client/httpapi/key.go b/client/httpapi/key.go
index dc2adf8b7..a16c30d8e 100644
--- a/client/httpapi/key.go
+++ b/client/httpapi/key.go
@@ -14,6 +14,8 @@ type KeyAPI HttpApi
 type keyOutput struct {
 	JName string `json:"Name"`
 	Id    string
+
+	pid peer.ID
 }
 
 func (k *keyOutput) Name() string {
@@ -26,13 +28,7 @@ func (k *keyOutput) Path() iface.Path {
 }
 
 func (k *keyOutput) ID() peer.ID {
-	p, _ := peer.IDB58Decode(k.Id)
-	return p
-}
-
-func (k *keyOutput) valid() error {
-	_, err := peer.IDB58Decode(k.Id)
-	return err
+	return k.pid
 }
 
 func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) {
@@ -49,7 +45,8 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key
 	if err != nil {
 		return nil, err
 	}
-	return &out, out.valid()
+	out.pid, err = peer.IDB58Decode(out.Id)
+	return &out, err
 }
 
 func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) {
@@ -72,7 +69,8 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o
 	}
 
 	id := &keyOutput{JName: out.Now, Id: out.Id}
-	return id, out.Overwrite, id.valid()
+	id.pid, err = peer.IDB58Decode(id.Id)
+	return id, out.Overwrite, err
 }
 
 func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
@@ -83,7 +81,9 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
 
 	res := make([]iface.Key, len(out.Keys))
 	for i, k := range out.Keys {
-		if err := k.valid(); err != nil {
+		var err error
+		k.pid, err = peer.IDB58Decode(k.Id)
+		if err != nil {
 			return nil, err
 		}
 		res[i] = k
@@ -98,8 +98,10 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
 		return nil, err
 	}
 
+	var err error
 	out := keyOutput{JName: "self", Id: id.ID}
-	return &out, out.valid()
+	out.pid, err = peer.IDB58Decode(out.Id)
+	return &out, err
 }
 
 func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) {
@@ -111,7 +113,9 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) {
 		return nil, errors.New("got unexpected number of keys back")
 	}
 
-	return &out.Keys[0], out.Keys[0].valid()
+	var err error
+	out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id)
+	return &out.Keys[0], err
 }
 
 func (api *KeyAPI) core() *HttpApi {
diff --git a/client/httpapi/name.go b/client/httpapi/name.go
index 58fb59249..b848aa819 100644
--- a/client/httpapi/name.go
+++ b/client/httpapi/name.go
@@ -16,10 +16,8 @@ type NameAPI HttpApi
 type ipnsEntry struct {
 	JName  string `json:"Name"`
 	JValue string `json:"Value"`
-}
 
-func (e *ipnsEntry) valid() (iface.Path, error) {
-	return iface.ParsePath(e.JValue)
+	path iface.Path
 }
 
 func (e *ipnsEntry) Name() string {
@@ -27,8 +25,7 @@ func (e *ipnsEntry) Name() string {
 }
 
 func (e *ipnsEntry) Value() iface.Path {
-	p, _ := e.valid()
-	return p
+	return e.path
 }
 
 func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) {
@@ -51,11 +48,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na
 	if err := req.Exec(ctx, &out); err != nil {
 		return nil, err
 	}
-	if _, err := out.valid(); err != nil {
-		return nil, err
-	}
-
-	return &out, nil
+	out.path, err = iface.ParsePath(out.JValue)
+	return &out, err
 }
 
 func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) {
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index 334509780..e27cb5655 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -66,16 +66,12 @@ type pubsubMessage struct {
 	JData     []byte   `json:"data,omitempty"`
 	JSeqno    []byte   `json:"seqno,omitempty"`
 	JTopicIDs []string `json:"topicIDs,omitempty"`
-}
 
-func (msg *pubsubMessage) valid() error {
-	_, err := peer.IDFromBytes(msg.JFrom)
-	return err
+	from peer.ID
 }
 
 func (msg *pubsubMessage) From() peer.ID {
-	id, _ := peer.IDFromBytes(msg.JFrom)
-	return id
+	return msg.from
 }
 
 func (msg *pubsubMessage) Data() []byte {
@@ -97,7 +93,9 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) {
 	if err := s.dec.Decode(&msg); err != nil {
 		return nil, err
 	}
-	return &msg, msg.valid()
+	var err error
+	msg.from, err = peer.IDFromBytes(msg.JFrom)
+	return &msg, err
 }
 
 func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) {

From 93f684617a3adbd7cba751ee421e32b75cd3dafa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 19:07:45 +0100
Subject: [PATCH 44/64] dag: remove unused waitgroup

This commit was moved from ipfs/go-ipfs-http-client@934fc60a7c41a14b679fd89fec3b349650a76c81
---
 client/httpapi/dag.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index 3f54ced34..a613d18de 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -39,13 +39,10 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error)
 
 func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption {
 	out := make(chan *format.NodeOption)
-	wg := sync.WaitGroup{}
-	wg.Add(len(cids))
 
 	for _, c := range cids {
 		// TODO: Consider limiting concurrency of this somehow
 		go func(c cid.Cid) {
-			defer wg.Done()
 			n, err := api.Get(ctx, c)
 
 			select {

From cc964b4ab8ea5d059be2c05e64ecdca851dcd8b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 19:15:21 +0100
Subject: [PATCH 45/64] Simplify Object.New, remove ipldnode.go

This commit was moved from ipfs/go-ipfs-http-client@0752a6ee63a2ec42aabb6c9acac6ef6c4a94af38
---
 client/httpapi/dag.go      |   1 -
 client/httpapi/ipldnode.go | 113 -------------------------------------
 client/httpapi/object.go   |  34 +++++------
 3 files changed, 18 insertions(+), 130 deletions(-)
 delete mode 100644 client/httpapi/ipldnode.go

diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go
index a613d18de..669b5f893 100644
--- a/client/httpapi/dag.go
+++ b/client/httpapi/dag.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"sync"
 
 	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go
deleted file mode 100644
index 43fa5d50d..000000000
--- a/client/httpapi/ipldnode.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package httpapi
-
-import (
-	"context"
-	"errors"
-	"io/ioutil"
-	"strconv"
-
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	ipfspath "github.com/ipfs/go-path"
-	"github.com/ipfs/interface-go-ipfs-core"
-)
-
-type ipldNode struct {
-	ctx  context.Context //TODO: should we re-consider adding ctx to ipld interfaces?
-	path iface.ResolvedPath
-	api  *HttpApi
-}
-
-func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node {
-	return &ipldNode{
-		ctx:  ctx,
-		path: p,
-		api:  api,
-	}
-}
-
-func (n *ipldNode) RawData() []byte {
-	r, err := n.api.Block().Get(n.ctx, n.path)
-	if err != nil {
-		panic(err) // TODO: eww, should we add errors too / better ideas?
-	}
-
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		panic(err)
-	}
-
-	return b
-}
-
-func (n *ipldNode) Cid() cid.Cid {
-	return n.path.Cid()
-}
-
-func (n *ipldNode) String() string {
-	return n.Cid().String()
-}
-
-func (n *ipldNode) Loggable() map[string]interface{} {
-	return nil //TODO: we can't really do better here, can we?
-}
-
-// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that.
-// We can also give people a choice
-func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) {
-	p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)})
-
-	var out interface{}
-	n.api.request("dag/get", p).Exec(n.ctx, &out)
-
-	// TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack
-	for len(path) > 0 {
-		switch o := out.(type) {
-		case map[string]interface{}:
-			v, ok := o[path[0]]
-			if !ok {
-				// TODO: ipld links
-				return nil, nil, errors.New("no element under this path")
-			}
-			out = v
-		case []interface{}:
-			n, err := strconv.ParseUint(path[0], 10, 32)
-			if err != nil {
-				return nil, nil, err
-			}
-			if len(o) < int(n) {
-				return nil, nil, errors.New("no element under this path")
-			}
-			out = o[n]
-		}
-		path = path[1:]
-	}
-
-	return out, path, nil
-}
-
-func (n *ipldNode) Tree(path string, depth int) []string {
-	panic("implement me")
-}
-
-func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) {
-	panic("implement me")
-}
-
-func (n *ipldNode) Copy() ipld.Node {
-	panic("implement me")
-}
-
-func (n *ipldNode) Links() []*ipld.Link {
-	panic("implement me")
-}
-
-func (n *ipldNode) Stat() (*ipld.NodeStat, error) {
-	panic("implement me")
-}
-
-func (n *ipldNode) Size() (uint64, error) {
-	panic("implement me")
-}
-
-var _ ipld.Node = &ipldNode{}
diff --git a/client/httpapi/object.go b/client/httpapi/object.go
index 3b648d82b..5a06f74d9 100644
--- a/client/httpapi/object.go
+++ b/client/httpapi/object.go
@@ -3,12 +3,15 @@ package httpapi
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"io/ioutil"
 
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipld-format"
+	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-merkledag"
+	dag "github.com/ipfs/go-merkledag"
+	ft "github.com/ipfs/go-unixfs"
 	"github.com/ipfs/interface-go-ipfs-core"
 	caopts "github.com/ipfs/interface-go-ipfs-core/options"
 )
@@ -19,24 +22,23 @@ type objectOut struct {
 	Hash string
 }
 
-func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) {
+func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) {
 	options, err := caopts.ObjectNewOptions(opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	var out objectOut
-	err = api.core().request("object/new", options.Type).Exec(ctx, &out)
-	if err != nil {
-		return nil, err
+	var n ipld.Node
+	switch options.Type {
+	case "empty":
+		n = new(dag.ProtoNode)
+	case "unixfs-dir":
+		n = ft.EmptyDirNode()
+	default:
+		return nil, fmt.Errorf("unknown object type: %s", options.Type)
 	}
 
-	c, err := cid.Parse(out.Hash)
-	if err != nil {
-		return nil, err
-	}
-
-	return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil
+	return n, nil
 }
 
 func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) {
@@ -64,7 +66,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec
 	return iface.IpfsPath(c), nil
 }
 
-func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) {
+func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) {
 	r, err := api.core().Block().Get(ctx, p)
 	if err != nil {
 		return nil, err
@@ -96,7 +98,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error)
 	return b, nil
 }
 
-func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) {
+func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) {
 	var out struct {
 		Links []struct {
 			Name string
@@ -107,14 +109,14 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link,
 	if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil {
 		return nil, err
 	}
-	res := make([]*format.Link, len(out.Links))
+	res := make([]*ipld.Link, len(out.Links))
 	for i, l := range out.Links {
 		c, err := cid.Parse(l.Hash)
 		if err != nil {
 			return nil, err
 		}
 
-		res[i] = &format.Link{
+		res[i] = &ipld.Link{
 			Cid:  c,
 			Name: l.Name,
 			Size: l.Size,

From a23da822de54922faf1f82a86c25660e8cca8bcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 14 Feb 2019 19:35:32 +0100
Subject: [PATCH 46/64] swarm: attach peerid if needed

This commit was moved from ipfs/go-ipfs-http-client@aa88b18ac497400d16b36af6489af50a003c298c
---
 client/httpapi/swarm.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
index d179b6540..0bb36aca3 100644
--- a/client/httpapi/swarm.go
+++ b/client/httpapi/swarm.go
@@ -17,6 +17,13 @@ type SwarmAPI HttpApi
 func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error {
 	saddrs := make([]string, len(pi.Addrs))
 	for i, addr := range pi.Addrs {
+		if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound {
+			pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty())
+			if err != nil {
+				return err
+			}
+			addr = addr.Encapsulate(pidma)
+		}
 		saddrs[i] = addr.String()
 	}
 

From f005b8dc5666916ff2a0ff1eef58b7795670ec73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 16:39:00 +0100
Subject: [PATCH 47/64] pin: verify: parse bad node cids early

This commit was moved from ipfs/go-ipfs-http-client@7032dfc9b016f2c584e2102e80fb60c3ecf573ea
---
 client/httpapi/pin.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index 4c4e5713c..b11efd35c 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -110,23 +110,18 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode {
 type badNode struct {
 	Cid  string
 	JErr string `json:"Err"`
+
+	cid cid.Cid
 }
 
 func (n *badNode) Path() iface.ResolvedPath {
-	c, err := cid.Parse(n.Cid)
-	if err != nil {
-		return nil // todo: handle this better
-	}
-	return iface.IpldPath(c)
+	return iface.IpldPath(n.cid)
 }
 
 func (n *badNode) Err() error {
 	if n.JErr != "" {
 		return errors.New(n.JErr)
 	}
-	if _, err := cid.Parse(n.Cid); err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -150,6 +145,13 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
 				return // todo: handle non io.EOF somehow
 			}
 
+			for i, n := range out.JBadNodes {
+				out.JBadNodes[i].cid, err = cid.Decode(n.Cid)
+				if err != nil {
+					return
+				}
+			}
+
 			select {
 			case res <- &out:
 			case <-ctx.Done():

From 4a6d36d98b6527e9c13918ac901c4cbc68630b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:02:13 +0100
Subject: [PATCH 48/64] pubsub: handle ctx

This commit was moved from ipfs/go-ipfs-http-client@d451a4943c5311d7e32fcf86f56bdc1eda977fc7
---
 client/httpapi/pubsub.go | 70 ++++++++++++++++++++++++++++++++--------
 1 file changed, 56 insertions(+), 14 deletions(-)

diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index e27cb5655..edc1a9709 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -57,8 +57,10 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte)
 }
 
 type pubsubSub struct {
-	io.Closer
-	dec *json.Decoder
+	messages chan pubsubMessage
+
+	done chan struct{}
+	rcloser io.Closer
 }
 
 type pubsubMessage struct {
@@ -68,6 +70,7 @@ type pubsubMessage struct {
 	JTopicIDs []string `json:"topicIDs,omitempty"`
 
 	from peer.ID
+	err error
 }
 
 func (msg *pubsubMessage) From() peer.ID {
@@ -87,15 +90,20 @@ func (msg *pubsubMessage) Topics() []string {
 }
 
 func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) {
-	// TODO: handle ctx
-
-	var msg pubsubMessage
-	if err := s.dec.Decode(&msg); err != nil {
-		return nil, err
+	select {
+	case msg, ok := <-s.messages:
+		if !ok {
+			return nil, io.EOF
+		}
+		if msg.err != nil {
+			return nil, msg.err
+		}
+		var err error
+		msg.from, err = peer.IDFromBytes(msg.JFrom)
+		return &msg, err
+	case <-ctx.Done():
+		return nil, ctx.Err()
 	}
-	var err error
-	msg.from, err = peer.IDFromBytes(msg.JFrom)
-	return &msg, err
 }
 
 func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) {
@@ -114,10 +122,44 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 		return nil, resp.Error
 	}
 
-	return &pubsubSub{
-		Closer: resp,
-		dec:    json.NewDecoder(resp.Output),
-	}, nil
+	sub := &pubsubSub{
+		messages: make(chan pubsubMessage),
+		done: make(chan struct{}),
+	}
+
+	dec := json.NewDecoder(resp.Output)
+
+	go func() {
+		defer close(sub.messages)
+
+		for {
+			var msg pubsubMessage
+			if err := dec.Decode(&msg); err != nil {
+				if err == io.EOF {
+					return
+				}
+				msg.err = err
+			}
+
+			select {
+			case sub.messages <- msg:
+			case <-sub.done:
+				return
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return sub, nil
+}
+
+func (s*pubsubSub) Close() error {
+	if s.done != nil {
+		close(s.done)
+		s.done = nil
+	}
+	return s.rcloser.Close()
 }
 
 func (api *PubsubAPI) core() *HttpApi {

From 7229dbbf73fd925da390fc5db5a25caf398b1fe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:04:33 +0100
Subject: [PATCH 49/64] don't read all and then throw away the buffer

This commit was moved from ipfs/go-ipfs-http-client@5c96c2954a5d45dd907dca9942d2773d76b8a71b
---
 client/httpapi/response.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 745c9a2a9..b9e83eb3d 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -139,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 		nresp.Output = nil
 
 		// drain body and close
-		ioutil.ReadAll(resp.Body)
+		io.Copy(ioutil.Discard, resp.Body)
 		resp.Body.Close()
 	}
 

From b6ace8dd401321b842c5f0054d9ed025fd19cd89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:12:37 +0100
Subject: [PATCH 50/64] response: handle late errors

This commit was moved from ipfs/go-ipfs-http-client@139e9e5ff1b52a92c169d28bf56ff58e36b49ca3
---
 client/httpapi/requestbuilder.go |  4 ++--
 client/httpapi/response.go       | 11 ++++++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index 628ad03cd..af43ce236 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -103,11 +103,11 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error {
 	}
 
 	if res == nil {
-		httpRes.Close()
+		lateErr := httpRes.Close()
 		if httpRes.Error != nil {
 			return httpRes.Error
 		}
-		return nil
+		return lateErr
 	}
 
 	return httpRes.Decode(res)
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index b9e83eb3d..f773130e8 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -39,9 +39,14 @@ type Response struct {
 
 func (r *Response) Close() error {
 	if r.Output != nil {
-		// always drain output (response body)
-		//ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data
-		return r.Output.Close()
+
+		// always drain output (response body) //TODO: make optional for things like cat
+		_, err1 := io.Copy(ioutil.Discard, r.Output)
+		err2 := r.Output.Close()
+		if err1 != nil {
+			return err1
+		}
+		return err2
 	}
 	return nil
 }

From 7be8d01ee732be71f08f4482e0d5c5b61731ee81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:27:12 +0100
Subject: [PATCH 51/64] response: option to disable output draining

This commit was moved from ipfs/go-ipfs-http-client@9f3d9635fa4e977b6ec202c9e4f78ca1c64fc0dd
---
 client/httpapi/api.go            |  1 +
 client/httpapi/apifile.go        |  2 +-
 client/httpapi/pubsub.go         |  4 ++--
 client/httpapi/request.go        |  2 ++
 client/httpapi/requestbuilder.go |  7 +++++++
 client/httpapi/response.go       | 10 ++++++++--
 6 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index e106d8d87..f74300216 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -131,6 +131,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 		command: command,
 		args:    args,
 		shell:   api,
+		drainOut: true,
 	}
 }
 
diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index 864760684..e3cb85ea4 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -57,7 +57,7 @@ func (f *apiFile) reset() error {
 	if f.r != nil {
 		f.r.Close()
 	}
-	req := f.core.request("cat", f.path.String())
+	req := f.core.request("cat", f.path.String()).NoDrain()
 	if f.at != 0 {
 		req.Option("offset", f.at)
 	}
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index edc1a9709..b3e45ed36 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -113,8 +113,8 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 	}
 
 	resp, err := api.core().request("pubsub/sub", topic).
-		Option("discover", options.Discover).
-		Send(ctx)
+		Option("discover", options.Discover).NoDrain().Send(ctx)
+
 	if err != nil {
 		return nil, err
 	}
diff --git a/client/httpapi/request.go b/client/httpapi/request.go
index 58c61ac67..18cfb7fd0 100644
--- a/client/httpapi/request.go
+++ b/client/httpapi/request.go
@@ -13,6 +13,7 @@ type Request struct {
 	Opts    map[string]string
 	Body    io.Reader
 	Headers map[string]string
+	DrainOut bool // if set, resp.Close will read all remaining data
 }
 
 func NewRequest(ctx context.Context, url, command string, args ...string) *Request {
@@ -30,5 +31,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque
 		Args:    args,
 		Opts:    opts,
 		Headers: make(map[string]string),
+		DrainOut: true,
 	}
 }
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index af43ce236..8b040522e 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -19,6 +19,7 @@ type RequestBuilder struct {
 	opts    map[string]string
 	headers map[string]string
 	body    io.Reader
+	drainOut bool
 
 	shell *HttpApi
 }
@@ -84,6 +85,12 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder {
 	return r
 }
 
+// NoDrain disables output draining in response closer
+func (r *RequestBuilder) NoDrain() *RequestBuilder {
+	r.drainOut = false
+	return r
+}
+
 // Send sends the request and return the response.
 func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) {
 	r.shell.applyGlobal(r)
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index f773130e8..cd3cc2b71 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -35,13 +35,18 @@ func (r *trailerReader) Close() error {
 type Response struct {
 	Output io.ReadCloser
 	Error  *Error
+
+	drainOutput bool
 }
 
 func (r *Response) Close() error {
 	if r.Output != nil {
 
-		// always drain output (response body) //TODO: make optional for things like cat
-		_, err1 := io.Copy(ioutil.Discard, r.Output)
+		// always drain output (response body)
+		var err1 error
+		if r.drainOutput {
+			_, err1 = io.Copy(ioutil.Discard, r.Output)
+		}
 		err2 := r.Output.Close()
 		if err1 != nil {
 			return err1
@@ -114,6 +119,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 
 	nresp := new(Response)
 
+	nresp.drainOutput = r.DrainOut
 	nresp.Output = &trailerReader{resp}
 	if resp.StatusCode >= http.StatusBadRequest {
 		e := &Error{

From 3088776f8150fe6967a245cf0360f5219e2dc9b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:30:02 +0100
Subject: [PATCH 52/64] swarm: always append peer IDs

This commit was moved from ipfs/go-ipfs-http-client@49267901c71f77f098deaf8966aa2dcecf22c2df
---
 client/httpapi/swarm.go | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
index 0bb36aca3..13a814bc4 100644
--- a/client/httpapi/swarm.go
+++ b/client/httpapi/swarm.go
@@ -15,16 +15,14 @@ import (
 type SwarmAPI HttpApi
 
 func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error {
+	pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty())
+	if err != nil {
+		return err
+	}
+
 	saddrs := make([]string, len(pi.Addrs))
 	for i, addr := range pi.Addrs {
-		if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound {
-			pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty())
-			if err != nil {
-				return err
-			}
-			addr = addr.Encapsulate(pidma)
-		}
-		saddrs[i] = addr.String()
+		saddrs[i] = addr.Encapsulate(pidma).String()
 	}
 
 	return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil)

From 745bf92506bd488bde4f695c4d644e7d3b58f0a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 18 Feb 2019 17:32:40 +0100
Subject: [PATCH 53/64] gofmt

This commit was moved from ipfs/go-ipfs-http-client@f3c2c350861470c05617cf74e7b78d06782bb706
---
 client/httpapi/api.go            |  6 +++---
 client/httpapi/pubsub.go         |  8 ++++----
 client/httpapi/request.go        | 22 +++++++++++-----------
 client/httpapi/requestbuilder.go | 10 +++++-----
 4 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index f74300216..6dbfe3b6a 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -128,9 +128,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
 
 func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 	return &RequestBuilder{
-		command: command,
-		args:    args,
-		shell:   api,
+		command:  command,
+		args:     args,
+		shell:    api,
 		drainOut: true,
 	}
 }
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index b3e45ed36..49c58ab88 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -59,7 +59,7 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte)
 type pubsubSub struct {
 	messages chan pubsubMessage
 
-	done chan struct{}
+	done    chan struct{}
 	rcloser io.Closer
 }
 
@@ -70,7 +70,7 @@ type pubsubMessage struct {
 	JTopicIDs []string `json:"topicIDs,omitempty"`
 
 	from peer.ID
-	err error
+	err  error
 }
 
 func (msg *pubsubMessage) From() peer.ID {
@@ -124,7 +124,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 
 	sub := &pubsubSub{
 		messages: make(chan pubsubMessage),
-		done: make(chan struct{}),
+		done:     make(chan struct{}),
 	}
 
 	dec := json.NewDecoder(resp.Output)
@@ -154,7 +154,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 	return sub, nil
 }
 
-func (s*pubsubSub) Close() error {
+func (s *pubsubSub) Close() error {
 	if s.done != nil {
 		close(s.done)
 		s.done = nil
diff --git a/client/httpapi/request.go b/client/httpapi/request.go
index 18cfb7fd0..d65d63835 100644
--- a/client/httpapi/request.go
+++ b/client/httpapi/request.go
@@ -7,12 +7,12 @@ import (
 )
 
 type Request struct {
-	ApiBase string
-	Command string
-	Args    []string
-	Opts    map[string]string
-	Body    io.Reader
-	Headers map[string]string
+	ApiBase  string
+	Command  string
+	Args     []string
+	Opts     map[string]string
+	Body     io.Reader
+	Headers  map[string]string
 	DrainOut bool // if set, resp.Close will read all remaining data
 }
 
@@ -26,11 +26,11 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque
 		"stream-channels": "true",
 	}
 	return &Request{
-		ApiBase: url + "/api/v0",
-		Command: command,
-		Args:    args,
-		Opts:    opts,
-		Headers: make(map[string]string),
+		ApiBase:  url + "/api/v0",
+		Command:  command,
+		Args:     args,
+		Opts:     opts,
+		Headers:  make(map[string]string),
 		DrainOut: true,
 	}
 }
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index 8b040522e..c5bc44124 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -14,11 +14,11 @@ import (
 
 // RequestBuilder is an IPFS commands request builder.
 type RequestBuilder struct {
-	command string
-	args    []string
-	opts    map[string]string
-	headers map[string]string
-	body    io.Reader
+	command  string
+	args     []string
+	opts     map[string]string
+	headers  map[string]string
+	body     io.Reader
 	drainOut bool
 
 	shell *HttpApi

From 27aa13fe44561e8d9469ce537e64baa60d17a705 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 21:53:17 +0100
Subject: [PATCH 54/64] return errors from constructor methods

This commit was moved from ipfs/go-ipfs-http-client@dbee4e27aaf9ef8e3fa7e25ac1f4f28df8a31e0d
---
 client/httpapi/api.go      | 39 ++++++++++++++++----------------------
 client/httpapi/api_test.go |  5 ++++-
 2 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 6dbfe3b6a..e4603bd38 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -29,8 +29,7 @@ type HttpApi struct {
 	applyGlobal func(*RequestBuilder)
 }
 
-//TODO: Return errors here
-func NewLocalApi() iface.CoreAPI {
+func NewLocalApi() (iface.CoreAPI, error) {
 	baseDir := os.Getenv(EnvDir)
 	if baseDir == "" {
 		baseDir = DefaultPathRoot
@@ -39,40 +38,34 @@ func NewLocalApi() iface.CoreAPI {
 	return NewPathApi(baseDir)
 }
 
-func NewPathApi(p string) iface.CoreAPI {
-	a := ApiAddr(p)
-	if a == nil {
-		return nil
+func NewPathApi(p string) (iface.CoreAPI, error) {
+	a, err := ApiAddr(p)
+	if err != nil {
+		if err == os.ErrNotExist {
+			err = nil
+		}
+		return nil, err
 	}
 	return NewApi(a)
 }
 
-func ApiAddr(ipfspath string) ma.Multiaddr {
+func ApiAddr(ipfspath string) (ma.Multiaddr, error) {
 	baseDir, err := homedir.Expand(ipfspath)
 	if err != nil {
-		return nil
+		return nil, err
 	}
 
 	apiFile := path.Join(baseDir, DefaultApiFile)
 
-	if _, err := os.Stat(apiFile); err != nil {
-		return nil
-	}
-
 	api, err := ioutil.ReadFile(apiFile)
 	if err != nil {
-		return nil
+		return nil, err
 	}
 
-	maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api)))
-	if err != nil {
-		return nil
-	}
-
-	return maddr
+	return ma.NewMultiaddr(strings.TrimSpace(string(api)))
 }
 
-func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr?
+func NewApi(a ma.Multiaddr) (*HttpApi, error) {
 	c := &gohttp.Client{
 		Transport: &gohttp.Transport{
 			Proxy:             gohttp.ProxyFromEnvironment,
@@ -83,10 +76,10 @@ func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr?
 	return NewApiWithClient(a, c)
 }
 
-func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
+func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) {
 	_, url, err := manet.DialArgs(a)
 	if err != nil {
-		return nil // TODO: return that error
+		return nil, err
 	}
 
 	if a, err := ma.NewMultiaddr(url); err == nil {
@@ -107,7 +100,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi {
 		return fmt.Errorf("unexpected redirect")
 	}
 
-	return api
+	return api, nil
 }
 
 func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) {
diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index af2180384..2f6d47c58 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -157,7 +157,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 					DisableCompression: true,
 				},
 			}
-			apis[i] = NewApiWithClient(maddr, c)
+			apis[i], err = NewApiWithClient(maddr, c)
+			if err != nil {
+				panic(err)
+			}
 
 			// empty node is pinned even with --empty-repo, we don't want that
 			emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")

From fc299e7e849d46e364204234f9ea3d3aa52e42cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 22:19:56 +0100
Subject: [PATCH 55/64] pin verify: use temporary struct

This commit was moved from ipfs/go-ipfs-http-client@e400fa3b380b589c2ae385a66a007d685147e4c7
---
 client/httpapi/pin.go | 57 +++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 21 deletions(-)

diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index b11efd35c..f667a5487 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -90,27 +90,20 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o
 }
 
 type pinVerifyRes struct {
-	Cid       string
-	JOk       bool       `json:"Ok"`
-	JBadNodes []*badNode `json:"BadNodes,omitempty"`
+	ok       bool
+	badNodes []iface.BadPinNode
 }
 
 func (r *pinVerifyRes) Ok() bool {
-	return r.JOk
+	return r.ok
 }
 
 func (r *pinVerifyRes) BadNodes() []iface.BadPinNode {
-	out := make([]iface.BadPinNode, len(r.JBadNodes))
-	for i, n := range r.JBadNodes {
-		out[i] = n
-	}
-	return out
+	return r.badNodes
 }
 
 type badNode struct {
-	Cid  string
-	JErr string `json:"Err"`
-
+	err error
 	cid cid.Cid
 }
 
@@ -119,10 +112,7 @@ func (n *badNode) Path() iface.ResolvedPath {
 }
 
 func (n *badNode) Err() error {
-	if n.JErr != "" {
-		return errors.New(n.JErr)
-	}
-	return nil
+	return n.err
 }
 
 func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
@@ -140,20 +130,45 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
 		defer close(res)
 		dec := json.NewDecoder(resp.Output)
 		for {
-			var out pinVerifyRes
+			var out struct {
+				Cid string
+				Ok bool
+
+				BadNodes []struct{
+					Cid string
+					Err string
+				}
+			}
 			if err := dec.Decode(&out); err != nil {
 				return // todo: handle non io.EOF somehow
 			}
 
-			for i, n := range out.JBadNodes {
-				out.JBadNodes[i].cid, err = cid.Decode(n.Cid)
+			badNodes := make([]iface.BadPinNode, len(out.BadNodes))
+			for i, n := range out.BadNodes {
+				c, err := cid.Decode(n.Cid)
 				if err != nil {
-					return
+					badNodes[i] = &badNode{
+						cid: c,
+						err: err,
+					}
+					continue
+				}
+
+				if n.Err != "" {
+					err = errors.New(n.Err)
+				}
+				badNodes[i] = &badNode{
+					cid: c,
+					err: err,
 				}
 			}
 
 			select {
-			case res <- &out:
+			case res <- &pinVerifyRes{
+				ok: out.Ok,
+
+				badNodes: badNodes,
+			}:
 			case <-ctx.Done():
 				return
 			}

From a058e7d31a32d1bca98e41886a310149b872af84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 22:27:48 +0100
Subject: [PATCH 56/64] response: Document zero-result Decode behaviour

This commit was moved from ipfs/go-ipfs-http-client@902bc5ef8c1b9595e97728dca3da0d0da124f142
---
 client/httpapi/response.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index cd3cc2b71..26f549c9a 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -71,6 +71,9 @@ func (r *Response) Decode(dec interface{}) error {
 		}
 		n++
 	}
+
+	// Decode expects at least one result. For calls where zero results are valid,
+	// use Send and construct json Decoder manually.
 	if n > 0 && err == io.EOF {
 		err = nil
 	}

From db1af499c9d661a70ac6fcce9cc7347b934385e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 22:33:57 +0100
Subject: [PATCH 57/64] use mime.ParseMediaType for Content-Type response
 parsing

This commit was moved from ipfs/go-ipfs-http-client@217d1945237321c12f96abef82707217a30813b0
---
 client/httpapi/response.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 26f549c9a..708d2714f 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -4,14 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/ipfs/go-ipfs-files"
 	"io"
 	"io/ioutil"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
-
-	files "github.com/ipfs/go-ipfs-files"
 )
 
 type trailerReader struct {
@@ -116,9 +115,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 		return nil, err
 	}
 
-	contentType := resp.Header.Get("Content-Type")
-	parts := strings.Split(contentType, ";")
-	contentType = parts[0]
+
+	contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, err
+	}
 
 	nresp := new(Response)
 

From f34788e6ee61aad0ee1970a4f3d004bdebdbcd27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 23:11:08 +0100
Subject: [PATCH 58/64] cleanup Swarm.Peers

This commit was moved from ipfs/go-ipfs-http-client@fd7858dc57b86761599b0e844f3536f2d4a0d953
---
 client/httpapi/api.go            |  1 -
 client/httpapi/apifile.go        | 14 +++---
 client/httpapi/pubsub.go         |  9 ++--
 client/httpapi/request.go        |  2 -
 client/httpapi/requestbuilder.go |  7 ---
 client/httpapi/response.go       | 19 +++++---
 client/httpapi/swarm.go          | 83 +++++++++++++++++---------------
 7 files changed, 69 insertions(+), 66 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index e4603bd38..c5a706d05 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -124,7 +124,6 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 		command:  command,
 		args:     args,
 		shell:    api,
-		drainOut: true,
 	}
 }
 
diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go
index e3cb85ea4..a8eb0de1a 100644
--- a/client/httpapi/apifile.go
+++ b/client/httpapi/apifile.go
@@ -49,15 +49,15 @@ type apiFile struct {
 	size int64
 	path iface.Path
 
-	r  io.ReadCloser
+	r  *Response
 	at int64
 }
 
 func (f *apiFile) reset() error {
 	if f.r != nil {
-		f.r.Close()
+		f.r.Cancel()
 	}
-	req := f.core.request("cat", f.path.String()).NoDrain()
+	req := f.core.request("cat", f.path.String())
 	if f.at != 0 {
 		req.Option("offset", f.at)
 	}
@@ -68,12 +68,12 @@ func (f *apiFile) reset() error {
 	if resp.Error != nil {
 		return resp.Error
 	}
-	f.r = resp.Output
+	f.r = resp
 	return nil
 }
 
 func (f *apiFile) Read(p []byte) (int, error) {
-	n, err := f.r.Read(p)
+	n, err := f.r.Output.Read(p)
 	if n > 0 {
 		f.at += int64(n)
 	}
@@ -92,7 +92,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
 	}
 
 	if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip
-		r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at)
+		r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at)
 
 		f.at += r
 		return f.at, err
@@ -103,7 +103,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
 
 func (f *apiFile) Close() error {
 	if f.r != nil {
-		return f.r.Close()
+		return f.r.Cancel()
 	}
 	return nil
 }
diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go
index 49c58ab88..2ac04b53c 100644
--- a/client/httpapi/pubsub.go
+++ b/client/httpapi/pubsub.go
@@ -60,7 +60,7 @@ type pubsubSub struct {
 	messages chan pubsubMessage
 
 	done    chan struct{}
-	rcloser io.Closer
+	rcloser func() error
 }
 
 type pubsubMessage struct {
@@ -113,7 +113,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 	}
 
 	resp, err := api.core().request("pubsub/sub", topic).
-		Option("discover", options.Discover).NoDrain().Send(ctx)
+		Option("discover", options.Discover).Send(ctx)
 
 	if err != nil {
 		return nil, err
@@ -125,6 +125,9 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt
 	sub := &pubsubSub{
 		messages: make(chan pubsubMessage),
 		done:     make(chan struct{}),
+		rcloser: func() error {
+			return resp.Cancel()
+		},
 	}
 
 	dec := json.NewDecoder(resp.Output)
@@ -159,7 +162,7 @@ func (s *pubsubSub) Close() error {
 		close(s.done)
 		s.done = nil
 	}
-	return s.rcloser.Close()
+	return s.rcloser()
 }
 
 func (api *PubsubAPI) core() *HttpApi {
diff --git a/client/httpapi/request.go b/client/httpapi/request.go
index d65d63835..f6f7ee486 100644
--- a/client/httpapi/request.go
+++ b/client/httpapi/request.go
@@ -13,7 +13,6 @@ type Request struct {
 	Opts     map[string]string
 	Body     io.Reader
 	Headers  map[string]string
-	DrainOut bool // if set, resp.Close will read all remaining data
 }
 
 func NewRequest(ctx context.Context, url, command string, args ...string) *Request {
@@ -31,6 +30,5 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque
 		Args:     args,
 		Opts:     opts,
 		Headers:  make(map[string]string),
-		DrainOut: true,
 	}
 }
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index c5bc44124..ba407217f 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -19,7 +19,6 @@ type RequestBuilder struct {
 	opts     map[string]string
 	headers  map[string]string
 	body     io.Reader
-	drainOut bool
 
 	shell *HttpApi
 }
@@ -85,12 +84,6 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder {
 	return r
 }
 
-// NoDrain disables output draining in response closer
-func (r *RequestBuilder) NoDrain() *RequestBuilder {
-	r.drainOut = false
-	return r
-}
-
 // Send sends the request and return the response.
 func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) {
 	r.shell.applyGlobal(r)
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 708d2714f..c9f178f9c 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -34,18 +34,13 @@ func (r *trailerReader) Close() error {
 type Response struct {
 	Output io.ReadCloser
 	Error  *Error
-
-	drainOutput bool
 }
 
 func (r *Response) Close() error {
 	if r.Output != nil {
 
-		// always drain output (response body)
-		var err1 error
-		if r.drainOutput {
-			_, err1 = io.Copy(ioutil.Discard, r.Output)
-		}
+		// drain output (response body)
+		_, err1 := io.Copy(ioutil.Discard, r.Output)
 		err2 := r.Output.Close()
 		if err1 != nil {
 			return err1
@@ -55,6 +50,15 @@ func (r *Response) Close() error {
 	return nil
 }
 
+// Cancel aborts running request (without draining request body)
+func (r *Response) Cancel() error {
+	if r.Output != nil {
+		return r.Output.Close()
+	}
+
+	return nil
+}
+
 func (r *Response) Decode(dec interface{}) error {
 	defer r.Close()
 	if r.Error != nil {
@@ -123,7 +127,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 
 	nresp := new(Response)
 
-	nresp.drainOutput = r.DrainOut
 	nresp.Output = &trailerReader{resp}
 	if resp.StatusCode >= http.StatusBadRequest {
 		e := &Error{
diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
index 13a814bc4..882711917 100644
--- a/client/httpapi/swarm.go
+++ b/client/httpapi/swarm.go
@@ -32,74 +32,81 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e
 	return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil)
 }
 
-type streamInfo struct {
-	Protocol string
-}
-
 type connInfo struct {
-	Addr       string
-	Peer       string
-	JLatency   time.Duration `json:"Latency"`
-	Muxer      string
-	JDirection inet.Direction `json:"Direction"`
-	JStreams   []streamInfo   `json:"Streams"`
-}
-
-func (c *connInfo) valid() error {
-	_, err := multiaddr.NewMultiaddr(c.Addr)
-	if err != nil {
-		return err
-	}
-
-	_, err = peer.IDB58Decode(c.Peer)
-	return err
+	addr       multiaddr.Multiaddr
+	peer       peer.ID
+	latency   time.Duration
+	muxer      string
+	direction inet.Direction
+	streams   []protocol.ID
 }
 
 func (c *connInfo) ID() peer.ID {
-	id, _ := peer.IDB58Decode(c.Peer)
-	return id
+	return c.peer
 }
 
 func (c *connInfo) Address() multiaddr.Multiaddr {
-	a, _ := multiaddr.NewMultiaddr(c.Addr)
-	return a
+	return c.addr
 }
 
 func (c *connInfo) Direction() inet.Direction {
-	return c.JDirection
+	return c.direction
 }
 
 func (c *connInfo) Latency() (time.Duration, error) {
-	return c.JLatency, nil
+	return c.latency, nil
 }
 
 func (c *connInfo) Streams() ([]protocol.ID, error) {
-	res := make([]protocol.ID, len(c.JStreams))
-	for i, stream := range c.JStreams {
-		res[i] = protocol.ID(stream.Protocol)
-	}
-	return res, nil
+	return c.streams, nil
 }
 
 func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) {
-	var out struct {
-		Peers []*connInfo
+	var resp struct {
+		Peers []struct{
+			Addr       string
+			Peer       string
+			Latency   time.Duration
+			Muxer      string
+			Direction inet.Direction
+			Streams   []struct {
+				Protocol string
+			}
+		}
 	}
 
 	err := api.core().request("swarm/peers").
 		Option("streams", true).
 		Option("latency", true).
-		Exec(ctx, &out)
+		Exec(ctx, &resp)
 	if err != nil {
 		return nil, err
 	}
 
-	res := make([]iface.ConnectionInfo, len(out.Peers))
-	for i, conn := range out.Peers {
-		if err := conn.valid(); err != nil {
+	res := make([]iface.ConnectionInfo, len(resp.Peers))
+	for i, conn := range resp.Peers {
+		out := &connInfo{
+			latency: conn.Latency,
+			muxer: conn.Muxer,
+			direction: conn.Direction,
+		}
+
+		out.peer, err = peer.IDB58Decode(conn.Peer)
+		if err != nil {
 			return nil, err
 		}
-		res[i] = conn
+
+		out.addr, err = multiaddr.NewMultiaddr(conn.Addr)
+		if err != nil {
+			return nil, err
+		}
+
+		out.streams = make([]protocol.ID, len(conn.Streams))
+		for i, p := range conn.Streams {
+			out.streams[i] = protocol.ID(p.Protocol)
+		}
+
+		res[i] = out
 	}
 
 	return res, nil

From aa4d6e16f13bd753b77d0a8fc2174e41af89310e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 23:26:34 +0100
Subject: [PATCH 59/64] Add some docs to constructors

This commit was moved from ipfs/go-ipfs-http-client@e34cd600e690bfa9bd20d6089b9335ac6090d4dd
---
 client/httpapi/api.go | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index c5a706d05..9e8d55188 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -22,6 +22,11 @@ const (
 	EnvDir          = "IPFS_PATH"
 )
 
+// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using
+// IPFS HTTP API.
+//
+// For interface docs see
+// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI
 type HttpApi struct {
 	url     string
 	httpcli gohttp.Client
@@ -29,6 +34,11 @@ type HttpApi struct {
 	applyGlobal func(*RequestBuilder)
 }
 
+// NewLocalApi tries to construct new HttpApi instance communicating with local
+// IPFS daemon
+//
+// Daemon api address is pulled from the $IPFS_PATH/api file.
+// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs
 func NewLocalApi() (iface.CoreAPI, error) {
 	baseDir := os.Getenv(EnvDir)
 	if baseDir == "" {
@@ -38,8 +48,10 @@ func NewLocalApi() (iface.CoreAPI, error) {
 	return NewPathApi(baseDir)
 }
 
-func NewPathApi(p string) (iface.CoreAPI, error) {
-	a, err := ApiAddr(p)
+// NewPathApi constructs new HttpApi by pulling api address from specified
+// ipfspath. Api file should be located at $ipfspath/api
+func NewPathApi(ipfspath string) (iface.CoreAPI, error) {
+	a, err := ApiAddr(ipfspath)
 	if err != nil {
 		if err == os.ErrNotExist {
 			err = nil
@@ -49,6 +61,7 @@ func NewPathApi(p string) (iface.CoreAPI, error) {
 	return NewApi(a)
 }
 
+// ApiAddr reads api file in specified ipfs path
 func ApiAddr(ipfspath string) (ma.Multiaddr, error) {
 	baseDir, err := homedir.Expand(ipfspath)
 	if err != nil {
@@ -65,6 +78,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) {
 	return ma.NewMultiaddr(strings.TrimSpace(string(api)))
 }
 
+// NewApi constructs HttpApi with specified endpoint
 func NewApi(a ma.Multiaddr) (*HttpApi, error) {
 	c := &gohttp.Client{
 		Transport: &gohttp.Transport{
@@ -76,6 +90,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) {
 	return NewApiWithClient(a, c)
 }
 
+// NewApiWithClient constructs HttpApi with specified endpoint and custom http client
 func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) {
 	_, url, err := manet.DialArgs(a)
 	if err != nil {

From f6a3a4fb2d715ffc0394408287b23eae0d322631 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 19 Feb 2019 23:28:39 +0100
Subject: [PATCH 60/64] gofmt

This commit was moved from ipfs/go-ipfs-http-client@65cd935e13a2ea05befbc69d72d64a89056952ea
---
 client/httpapi/api.go            |  6 +++---
 client/httpapi/pin.go            |  4 ++--
 client/httpapi/request.go        | 22 +++++++++++-----------
 client/httpapi/requestbuilder.go | 10 +++++-----
 client/httpapi/response.go       |  1 -
 client/httpapi/swarm.go          | 18 +++++++++---------
 6 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index 9e8d55188..b67fb0b48 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -136,9 +136,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
 
 func (api *HttpApi) request(command string, args ...string) *RequestBuilder {
 	return &RequestBuilder{
-		command:  command,
-		args:     args,
-		shell:    api,
+		command: command,
+		args:    args,
+		shell:   api,
 	}
 }
 
diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go
index f667a5487..0111d626a 100644
--- a/client/httpapi/pin.go
+++ b/client/httpapi/pin.go
@@ -132,9 +132,9 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
 		for {
 			var out struct {
 				Cid string
-				Ok bool
+				Ok  bool
 
-				BadNodes []struct{
+				BadNodes []struct {
 					Cid string
 					Err string
 				}
diff --git a/client/httpapi/request.go b/client/httpapi/request.go
index f6f7ee486..58c61ac67 100644
--- a/client/httpapi/request.go
+++ b/client/httpapi/request.go
@@ -7,12 +7,12 @@ import (
 )
 
 type Request struct {
-	ApiBase  string
-	Command  string
-	Args     []string
-	Opts     map[string]string
-	Body     io.Reader
-	Headers  map[string]string
+	ApiBase string
+	Command string
+	Args    []string
+	Opts    map[string]string
+	Body    io.Reader
+	Headers map[string]string
 }
 
 func NewRequest(ctx context.Context, url, command string, args ...string) *Request {
@@ -25,10 +25,10 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque
 		"stream-channels": "true",
 	}
 	return &Request{
-		ApiBase:  url + "/api/v0",
-		Command:  command,
-		Args:     args,
-		Opts:     opts,
-		Headers:  make(map[string]string),
+		ApiBase: url + "/api/v0",
+		Command: command,
+		Args:    args,
+		Opts:    opts,
+		Headers: make(map[string]string),
 	}
 }
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index ba407217f..af43ce236 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -14,11 +14,11 @@ import (
 
 // RequestBuilder is an IPFS commands request builder.
 type RequestBuilder struct {
-	command  string
-	args     []string
-	opts     map[string]string
-	headers  map[string]string
-	body     io.Reader
+	command string
+	args    []string
+	opts    map[string]string
+	headers map[string]string
+	body    io.Reader
 
 	shell *HttpApi
 }
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index c9f178f9c..84f28214d 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -119,7 +119,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 		return nil, err
 	}
 
-
 	contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
 	if err != nil {
 		return nil, err
diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go
index 882711917..0814debee 100644
--- a/client/httpapi/swarm.go
+++ b/client/httpapi/swarm.go
@@ -33,10 +33,10 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e
 }
 
 type connInfo struct {
-	addr       multiaddr.Multiaddr
-	peer       peer.ID
+	addr      multiaddr.Multiaddr
+	peer      peer.ID
 	latency   time.Duration
-	muxer      string
+	muxer     string
 	direction inet.Direction
 	streams   []protocol.ID
 }
@@ -63,11 +63,11 @@ func (c *connInfo) Streams() ([]protocol.ID, error) {
 
 func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) {
 	var resp struct {
-		Peers []struct{
-			Addr       string
-			Peer       string
+		Peers []struct {
+			Addr      string
+			Peer      string
 			Latency   time.Duration
-			Muxer      string
+			Muxer     string
 			Direction inet.Direction
 			Streams   []struct {
 				Protocol string
@@ -86,8 +86,8 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error)
 	res := make([]iface.ConnectionInfo, len(resp.Peers))
 	for i, conn := range resp.Peers {
 		out := &connInfo{
-			latency: conn.Latency,
-			muxer: conn.Muxer,
+			latency:   conn.Latency,
+			muxer:     conn.Muxer,
 			direction: conn.Direction,
 		}
 

From ad844e3d0b0221333c04896ed80f11afed301495 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 20 Feb 2019 16:50:38 +0100
Subject: [PATCH 61/64] response: simplify Decode

This commit was moved from ipfs/go-ipfs-http-client@adbfda4c1c6ee037df3f91ca73f60de1f4620d67
---
 client/httpapi/api.go            |  2 +-
 client/httpapi/requestbuilder.go |  2 +-
 client/httpapi/response.go       | 20 +++-----------------
 3 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/client/httpapi/api.go b/client/httpapi/api.go
index b67fb0b48..17a289cc5 100644
--- a/client/httpapi/api.go
+++ b/client/httpapi/api.go
@@ -53,7 +53,7 @@ func NewLocalApi() (iface.CoreAPI, error) {
 func NewPathApi(ipfspath string) (iface.CoreAPI, error) {
 	a, err := ApiAddr(ipfspath)
 	if err != nil {
-		if err == os.ErrNotExist {
+		if os.IsNotExist(err) {
 			err = nil
 		}
 		return nil, err
diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go
index af43ce236..2ffed7a0a 100644
--- a/client/httpapi/requestbuilder.go
+++ b/client/httpapi/requestbuilder.go
@@ -110,5 +110,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error {
 		return lateErr
 	}
 
-	return httpRes.Decode(res)
+	return httpRes.decode(res)
 }
diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 84f28214d..794cd1470 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -59,28 +59,14 @@ func (r *Response) Cancel() error {
 	return nil
 }
 
-func (r *Response) Decode(dec interface{}) error {
+// Decode reads request body and decodes it as json
+func (r *Response) decode(dec interface{}) error {
 	defer r.Close()
 	if r.Error != nil {
 		return r.Error
 	}
 
-	n := 0
-	var err error
-	for {
-		err = json.NewDecoder(r.Output).Decode(dec)
-		if err != nil {
-			break
-		}
-		n++
-	}
-
-	// Decode expects at least one result. For calls where zero results are valid,
-	// use Send and construct json Decoder manually.
-	if n > 0 && err == io.EOF {
-		err = nil
-	}
-	return err
+	return json.NewDecoder(r.Output).Decode(dec)
 }
 
 type Error struct {

From 19d91fbb259220e91c8887ee8a09218865418720 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Wed, 20 Feb 2019 22:08:02 +0100
Subject: [PATCH 62/64] response: pass close error in decode

This commit was moved from ipfs/go-ipfs-http-client@4a87232ecaa96246f038d15eb7ee6094216a201e
---
 client/httpapi/response.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 794cd1470..624a488d7 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -61,12 +61,17 @@ func (r *Response) Cancel() error {
 
 // Decode reads request body and decodes it as json
 func (r *Response) decode(dec interface{}) error {
-	defer r.Close()
 	if r.Error != nil {
 		return r.Error
 	}
 
-	return json.NewDecoder(r.Output).Decode(dec)
+	err := json.NewDecoder(r.Output).Decode(dec)
+	err2 := r.Close()
+	if err != nil {
+		return err
+	}
+
+	return err2
 }
 
 type Error struct {

From cc9968d5dda71c21b3c90eb376458cacadb59e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 21 Feb 2019 14:39:12 +0100
Subject: [PATCH 63/64] request: fix Content-Disposition header in Send

This commit was moved from ipfs/go-ipfs-http-client@b7db17c63b69dc4fbb726dc720e9c594fa9ed495
---
 client/httpapi/response.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/httpapi/response.go b/client/httpapi/response.go
index 624a488d7..339c73658 100644
--- a/client/httpapi/response.go
+++ b/client/httpapi/response.go
@@ -102,7 +102,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 
 	if fr, ok := r.Body.(*files.MultiFileReader); ok {
 		req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary())
-		req.Header.Set("Content-Disposition", "form-data: name=\"files\"")
+		req.Header.Set("Content-Disposition", "form-data; name=\"files\"")
 	}
 
 	resp, err := c.Do(req)

From 47b820150bddb9868274ed2c6faf79c6b8cbe369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Thu, 21 Feb 2019 14:46:18 +0100
Subject: [PATCH 64/64] test: don't panic on errors in async node construction

This commit was moved from ipfs/go-ipfs-http-client@95ce0f3949da47b2db92508b8c7b0a00f502682b
---
 client/httpapi/api_test.go | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go
index 2f6d47c58..df45c15af 100644
--- a/client/httpapi/api_test.go
+++ b/client/httpapi/api_test.go
@@ -114,27 +114,32 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 
 	wg.Add(len(nodes))
 	zero.Add(1)
+	errs := make(chan error, len(nodes))
 
 	for i, nd := range nodes {
 		go func(i int, nd testbedi.Core) {
 			defer wg.Done()
 
 			if _, err := nd.Init(ctx, "--empty-repo"); err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			if i > 0 {
 				zero.Wait()
 				if err := nd.Connect(ctx, nodes[0]); err != nil {
-					panic(err)
+					errs <- err
+					return
 				}
 			} else {
 				zero.Done()
@@ -142,12 +147,14 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 
 			addr, err := nd.APIAddr()
 			if err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			maddr, err := ma.NewMultiaddr(addr)
 			if err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			c := &gohttp.Client{
@@ -159,16 +166,19 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 			}
 			apis[i], err = NewApiWithClient(maddr, c)
 			if err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 
 			// empty node is pinned even with --empty-repo, we don't want that
 			emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
 			if err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 			if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil {
-				panic(err)
+				errs <- err
+				return
 			}
 		}(i, nd)
 	}
@@ -187,7 +197,12 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		}()
 	}()
 
-	return apis, nil
+	select {
+	case err = <-errs:
+	default:
+	}
+
+	return apis, err
 }
 
 func TestHttpApi(t *testing.T) {