coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-03-07 01:08:08 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

cache encoded data when reading dag nodes from disk

Browse Source 
License: MIT
Signed-off-by: Jeromy <why@ipfs.io>
This commit is contained in:
Jeromy 2016-07-10 00:40:23 -07:00
parent fd971ed934
commit bf23516dc0
36 changed files with 144 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/commands/add.go
View File

@ -12,6 +12,7 @@ import (
core "github.com/ipfs/go-ipfs/core"
dagtest "github.com/ipfs/go-ipfs/merkledag/test"
mfs "github.com/ipfs/go-ipfs/mfs"
ft "github.com/ipfs/go-ipfs/unixfs"
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)
@ -160,7 +161,7 @@ You can now refer to the added file in a gateway, like so:
if hash {
md := dagtest.Mock()
mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return

4
core/commands/files/files.go
View File

@ -163,7 +163,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
return nil, err
}
d, err := ft.FromBytes(nd.Data)
d, err := ft.FromBytes(nd.Data())
if err != nil {
return nil, err
}
@ -871,7 +871,7 @@ func getFileHandle(r *mfs.Root, path string, create bool) (*mfs.File, error) {
return nil, fmt.Errorf("%s was not a directory", dirname)
}
nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
nd := dag.NodeWithData(ft.FilePBData(nil, 0))
err = pdir.AddChild(fname, nd)
if err != nil {
return nil, err

2
core/commands/ls.go
View File

@ -110,7 +110,7 @@ format:
}
}
if linkNode != nil {
d, err := unixfs.FromBytes(linkNode.Data)
d, err := unixfs.FromBytes(linkNode.Data())
if err != nil {
res.SetError(err, cmds.ErrNormal)
return

13
core/commands/object/object.go
View File

@ -93,7 +93,7 @@ is the raw data of the object.
res.SetError(err, cmds.ErrNormal)
return
}
res.SetOutput(bytes.NewReader(node.Data))
res.SetOutput(bytes.NewReader(node.Data()))
},
}
@ -198,7 +198,7 @@ This command outputs data in the following encodings:
node := &Node{
Links: make([]Link, len(object.Links)),
Data: string(object.Data),
Data: string(object.Data()),
}
for i, link := range object.Links {
@ -438,9 +438,7 @@ Available templates:
func nodeFromTemplate(template string) (*dag.Node, error) {
switch template {
case "unixfs-dir":
nd := new(dag.Node)
nd.Data = ft.FolderPBData()
return nd, nil
return ft.EmptyDirNode(), nil
default:
return nil, fmt.Errorf("template '%s' not found", template)
}
@ -566,9 +564,10 @@ func deserializeNode(node *Node, dataFieldEncoding string) (*dag.Node, error) {
dagnode := new(dag.Node)
switch dataFieldEncoding {
case "text":
dagnode.Data = []byte(node.Data)
dagnode.SetData([]byte(node.Data))
case "base64":
dagnode.Data, _ = base64.StdEncoding.DecodeString(node.Data)
data, _ := base64.StdEncoding.DecodeString(node.Data)
dagnode.SetData(data)
default:
return nil, fmt.Errorf("Unkown data field encoding")
}

8
core/commands/object/patch.go
View File

@ -91,7 +91,7 @@ the limit will not be respected by the network.
return
}
rootnd.Data = append(rootnd.Data, data...)
rootnd.SetData(append(rootnd.Data(), data...))
newkey, err := nd.DAG.Add(rootnd)
if err != nil {
@ -153,7 +153,7 @@ Example:
return
}
root.Data = data
root.SetData(data)
newkey, err := nd.DAG.Add(root)
if err != nil {
@ -287,9 +287,7 @@ to a file containing 'bar', and returns the hash of the new object.
var createfunc func() *dag.Node
if create {
createfunc = func() *dag.Node {
return &dag.Node{Data: ft.FolderPBData()}
}
createfunc = ft.EmptyDirNode
}
e := dagutils.NewDagEditor(root, nd.DAG)

4
core/commands/unixfs/ls.go
View File

@ -101,7 +101,7 @@ Example:
continue
}
unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
unixFSNode, err := unixfs.FromBytes(merkleNode.Data())
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
@ -128,7 +128,7 @@ Example:
res.SetError(err, cmds.ErrNormal)
return
}
d, err := unixfs.FromBytes(linkNode.Data)
d, err := unixfs.FromBytes(linkNode.Data())
if err != nil {
res.SetError(err, cmds.ErrNormal)
return

2
core/corehttp/gateway_handler.go
View File

@ -391,7 +391,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
case nil:
// object set-data case
rnode.Data = newnode.Data
rnode.SetData(newnode.Data())
newkey, err = i.node.DAG.Add(rnode)
if err != nil {

11
core/coreunix/add.go
View File

@ -29,8 +29,6 @@ import (
var log = logging.Logger("coreunix")
var folderData = unixfs.FolderPBData()
// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256
@ -67,7 +65,7 @@ type AddedObject struct {
}
func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil)
if err != nil {
return nil, err
}
@ -392,7 +390,7 @@ func (adder *Adder) addFile(file files.File) error {
return err
}
dagnode := &dag.Node{Data: sdata}
dagnode := dag.NodeWithData(sdata)
_, err = adder.dagService.Add(dagnode)
if err != nil {
return err
@ -488,11 +486,6 @@ func NewMemoryDagService() dag.DAGService {
return dag.NewDAGService(bsrv)
}
// TODO: generalize this to more than unix-fs nodes.
func NewDirNode() *dag.Node {
return &dag.Node{Data: unixfs.FolderPBData()}
}
// from core/commands/object.go
func getOutput(dagnode *dag.Node) (*Object, error) {
key, err := dagnode.Key()

4
core/coreunix/metadata.go
View File

@ -21,7 +21,7 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error
return "", err
}
mdnode.Data = mdata
mdnode.SetData(mdata)
if err := mdnode.AddNodeLinkClean("file", nd); err != nil {
return "", err
}
@ -42,5 +42,5 @@ func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) {
return nil, err
}
return ft.MetadataFromBytes(nd.Data)
return ft.MetadataFromBytes(nd.Data())
}

3
fuse/ipns/common.go
View File

@ -4,7 +4,6 @@ import (
context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
"github.com/ipfs/go-ipfs/core"
mdag "github.com/ipfs/go-ipfs/merkledag"
nsys "github.com/ipfs/go-ipfs/namesys"
path "github.com/ipfs/go-ipfs/path"
ft "github.com/ipfs/go-ipfs/unixfs"
@ -14,7 +13,7 @@ import (
// InitializeKeyspace sets the ipns record for the given key to
// point to an empty directory.
func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
emptyDir := &mdag.Node{Data: ft.FolderPBData()}
emptyDir := ft.EmptyDirNode()
nodek, err := n.DAG.Add(emptyDir)
if err != nil {
return err

2
fuse/ipns/ipns_unix.go
View File

@ -481,7 +481,7 @@ func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
// New 'empty' file
nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
nd := dag.NodeWithData(ft.FilePBData(nil, 0))
err := dir.dir.AddChild(req.Name, nd)
if err != nil {
return nil, nil, err

2
fuse/readonly/readonly_unix.go
View File

@ -84,7 +84,7 @@ type Node struct {
func (s *Node) loadData() error {
s.cached = new(ftpb.Data)
return proto.Unmarshal(s.Nd.Data, s.cached)
return proto.Unmarshal(s.Nd.Data(), s.cached)
}
// Attr returns the attributes of a given node.

4
importer/helpers/helpers.go
View File

@ -59,7 +59,7 @@ func NewUnixfsBlock() *UnixfsNode {
// NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
mb, err := ft.FSNodeFromBytes(nd.Data)
mb, err := ft.FSNodeFromBytes(nd.Data())
if err != nil {
return nil, err
}
@ -126,6 +126,6 @@ func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
if err != nil {
return nil, err
}
n.node.Data = data
n.node.SetData(data)
return n.node, nil
}

4
importer/trickle/trickle_test.go
View File

@ -524,7 +524,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
data := []byte("AB")
nd := new(merkledag.Node)
nd.Data = ft.FilePBData(nil, 0)
nd.SetData(ft.FilePBData(nil, 0))
dbp := &h.DagBuilderParams{
Dagserv: ds,
@ -562,7 +562,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
}
func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
pbd, err := ft.FromBytes(nd.Data)
pbd, err := ft.FromBytes(nd.Data())
if err != nil {
panic(err)
}

4
importer/trickle/trickledag.go
View File

@ -241,7 +241,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
return errors.New("expected direct block")
}
pbn, err := ft.FromBytes(nd.Data)
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return err
}
@ -253,7 +253,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
}
// Verify this is a branch node
pbn, err := ft.FromBytes(nd.Data)
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return err
}

11
merkledag/coding.go
View File

@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error {
}
sort.Stable(LinkSlice(n.Links)) // keep links sorted
n.Data = pbn.GetData()
n.data = pbn.GetData()
n.encoded = encoded
return nil
}
@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode {
pbn.Links[i].Hash = []byte(l.Hash)
}
if len(n.Data) > 0 {
pbn.Data = n.Data
if len(n.data) > 0 {
pbn.Data = n.data
}
return pbn
}
@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode {
func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
sort.Stable(LinkSlice(n.Links)) // keep links sorted
if n.encoded == nil || force {
n.cached = nil
var err error
n.encoded, err = n.Marshal()
if err != nil {
return nil, err
}
}
if n.cached == nil {
n.cached = u.Hash(n.encoded)
}

4
merkledag/merkledag.go
View File

@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
}
return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
}
res.cached = k.ToMultihash()
return res, nil
}
@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
out <- &NodeOption{Err: err}
return
}
nd.cached = b.Key().ToMultihash()
// buffered, no need to select
out <- &NodeOption{Node: nd}

18
merkledag/merkledag_test.go
View File

@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
func TestNode(t *testing.T) {
n1 := &Node{Data: []byte("beep")}
n2 := &Node{Data: []byte("boop")}
n3 := &Node{Data: []byte("beep boop")}
n1 := NodeWithData([]byte("beep"))
n2 := NodeWithData([]byte("boop"))
n3 := NodeWithData([]byte("beep boop"))
if err := n3.AddNodeLink("beep-link", n1); err != nil {
t.Error(err)
}
@ -58,7 +58,7 @@ func TestNode(t *testing.T) {
printn := func(name string, n *Node) {
fmt.Println(">", name)
fmt.Println("data:", string(n.Data))
fmt.Println("data:", string(n.Data()))
fmt.Println("links:")
for _, l := range n.Links {
@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) {
expected := NodeStat{
NumLinks: len(n.Links),
BlockSize: len(enc),
LinksSize: len(enc) - len(n.Data), // includes framing.
DataSize: len(n.Data),
LinksSize: len(enc) - len(n.Data()), // includes framing.
DataSize: len(n.Data()),
CumulativeSize: int(cumSize),
Hash: k.B58String(),
}
@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) {
func TestCantGet(t *testing.T) {
dsp := getDagservAndPinner(t)
a := &Node{Data: []byte("A")}
a := NodeWithData([]byte("A"))
k, err := a.Key()
if err != nil {
@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) {
top := new(Node)
for i := 0; i < 10; i++ {
nd := &Node{Data: []byte{byte('a' + i)}}
nd := NodeWithData([]byte{byte('a' + i)})
_, err := ds.Add(nd)
if err != nil {
t.Fatal(err)
@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) {
}
for i := 0; i < 10; i++ {
nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
nd := NodeWithData([]byte{'f', 'a' + byte(i)})
_, err := ds_bad.Add(nd)
if err != nil {
t.Fatal(err)

26
merkledag/node.go
View File

@ -15,7 +15,7 @@ var ErrLinkNotFound = fmt.Errorf("no link by that name")
// nodes have opaque data and a set of navigable links.
type Node struct {
Links []*Link
Data []byte
data []byte
// cache encoded/marshaled value
encoded []byte
@ -78,6 +78,10 @@ func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
return serv.Get(ctx, key.Key(l.Hash))
}
func NodeWithData(d []byte) *Node {
return &Node{data: d}
}
// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
n.encoded = nil
@ -168,9 +172,9 @@ func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*
// NOTE: Does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
nnode := new(Node)
if len(n.Data) > 0 {
nnode.Data = make([]byte, len(n.Data))
copy(nnode.Data, n.Data)
if len(n.data) > 0 {
nnode.data = make([]byte, len(n.data))
copy(nnode.data, n.data)
}
if len(n.Links) > 0 {
@ -180,6 +184,16 @@ func (n *Node) Copy() *Node {
return nnode
}
func (n *Node) Data() []byte {
return n.data
}
func (n *Node) SetData(d []byte) {
n.encoded = nil
n.cached = nil
n.data = d
}
// UpdateNodeLink return a copy of the node with the link name set to point to
// that. If a link of the same name existed, it is removed.
func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) {
@ -226,8 +240,8 @@ func (n *Node) Stat() (*NodeStat, error) {
Hash: key.B58String(),
NumLinks: len(n.Links),
BlockSize: len(enc),
LinksSize: len(enc) - len(n.Data), // includes framing.
DataSize: len(n.Data),
LinksSize: len(enc) - len(n.data), // includes framing.
DataSize: len(n.data),
CumulativeSize: int(cumSize),
}, nil
}

14
merkledag/traverse/traverse_test.go
View File

@ -326,7 +326,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
buf := new(bytes.Buffer)
walk := func(current State) error {
s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data)
s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data())
t.Logf("walk: %s", s)
buf.Write([]byte(s))
return nil
@ -349,7 +349,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
}
func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
a := mdag.NodeWithData([]byte("/a"))
addLink(t, ds, a, child(t, ds, a, "aa"))
addLink(t, ds, a, child(t, ds, a, "ab"))
addLink(t, ds, a, child(t, ds, a, "ac"))
@ -358,7 +358,7 @@ func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
}
func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
aaaa := child(t, ds, aaa, "aaaa")
@ -371,7 +371,7 @@ func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
}
func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
ab := child(t, ds, a, "ab")
addLink(t, ds, aa, child(t, ds, aa, "aaa"))
@ -384,7 +384,7 @@ func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
}
func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
aaaa := child(t, ds, aaa, "aaaa")
@ -401,7 +401,7 @@ func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
}
func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
to := string(a.Data) + "2" + string(b.Data)
to := string(a.Data()) + "2" + string(b.Data())
if _, err := ds.Add(b); err != nil {
t.Error(err)
}
@ -411,5 +411,5 @@ func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
}
func child(t *testing.T, ds mdag.DAGService, a *mdag.Node, name string) *mdag.Node {
return &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
return mdag.NodeWithData([]byte(string(a.Data()) + "/" + name))
}

6
merkledag/utils/utils_test.go
View File

@ -13,9 +13,7 @@ import (
func TestAddLink(t *testing.T) {
ds := mdtest.Mock()
fishnode := &dag.Node{
Data: []byte("fishcakes!"),
}
fishnode := dag.NodeWithData([]byte("fishcakes!"))
fk, err := ds.Add(fishnode)
if err != nil {
@ -90,7 +88,7 @@ func TestInsertNode(t *testing.T) {
}
func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
child := &dag.Node{Data: []byte(data)}
child := dag.NodeWithData([]byte(data))
ck, err := e.tmp.Add(child)
if err != nil {
t.Fatal(err)

5
mfs/dir.go
View File

@ -121,7 +121,7 @@ func (d *Directory) childNode(name string) (FSNode, error) {
// cacheNode caches a node into d.childDirs or d.files and returns the FSNode.
func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) {
i, err := ft.FromBytes(nd.Data)
i, err := ft.FromBytes(nd.Data())
if err != nil {
return nil, err
}
@ -268,7 +268,8 @@ func (d *Directory) Mkdir(name string) (*Directory, error) {
}
}
ndir := &dag.Node{Data: ft.FolderPBData()}
ndir := new(dag.Node)
ndir.SetData(ft.FolderPBData())
_, err = d.dserv.Add(ndir)
if err != nil {

4
mfs/file.go
View File

@ -45,7 +45,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
node := fi.node
fi.nodelk.Unlock()
fsn, err := ft.FSNodeFromBytes(node.Data)
fsn, err := ft.FSNodeFromBytes(node.Data())
if err != nil {
return nil, err
}
@ -86,7 +86,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
func (fi *File) Size() (int64, error) {
fi.nodelk.Lock()
defer fi.nodelk.Unlock()
pbd, err := ft.FromBytes(fi.node.Data)
pbd, err := ft.FromBytes(fi.node.Data())
if err != nil {
return 0, err
}

36
mfs/mfs_test.go
View File

@ -31,6 +31,10 @@ import (
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)
func emptyDirNode() *dag.Node {
return dag.NodeWithData(ft.FolderPBData())
}
func getDagserv(t *testing.T) dag.DAGService {
db := dssync.MutexWrap(ds.NewMapDatastore())
bs := bstore.NewBlockstore(db)
@ -183,7 +187,7 @@ func catNode(ds dag.DAGService, nd *dag.Node) ([]byte, error) {
func setupRoot(ctx context.Context, t *testing.T) (dag.DAGService, *Root) {
ds := getDagserv(t)
root := &dag.Node{Data: ft.FolderPBData()}
root := emptyDirNode()
rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, k key.Key) error {
fmt.Println("PUBLISHED: ", k)
return nil
@ -282,7 +286,7 @@ func TestDirectoryLoadFromDag(t *testing.T) {
t.Fatal(err)
}
dir := &dag.Node{Data: ft.FolderPBData()}
dir := emptyDirNode()
_, err = ds.Add(dir)
if err != nil {
t.Fatal(err)
@ -293,17 +297,15 @@ func TestDirectoryLoadFromDag(t *testing.T) {
t.Fatal(err)
}
top := &dag.Node{
Data: ft.FolderPBData(),
Links: []*dag.Link{
&dag.Link{
Name: "a",
Hash: fihash,
},
&dag.Link{
Name: "b",
Hash: dirhash,
},
top := emptyDirNode()
top.Links = []*dag.Link{
&dag.Link{
Name: "a",
Hash: fihash,
},
&dag.Link{
Name: "b",
Hash: dirhash,
},
}
@ -540,7 +542,7 @@ func actorMakeFile(d *Directory) error {
}
name := randomName()
f, err := NewFile(name, &dag.Node{Data: ft.FilePBData(nil, 0)}, d, d.dserv)
f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dserv)
if err != nil {
return err
}
@ -756,7 +758,7 @@ func TestFlushing(t *testing.T) {
e := mkdirP(t, dir, "a/b/e")
data := []byte("this is a test\n")
nd1 := &dag.Node{Data: ft.FilePBData(data, uint64(len(data)))}
nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data))))
if err := c.AddChild("TEST", nd1); err != nil {
t.Fatal(err)
@ -792,7 +794,7 @@ func TestFlushing(t *testing.T) {
t.Fatal(err)
}
fsnode, err := ft.FSNodeFromBytes(rnd.Data)
fsnode, err := ft.FSNodeFromBytes(rnd.Data())
if err != nil {
t.Fatal(err)
}
@ -897,7 +899,7 @@ func TestFileDescriptors(t *testing.T) {
ds, rt := setupRoot(ctx, t)
dir := rt.GetValue().(*Directory)
nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
nd := dag.NodeWithData(ft.FilePBData(nil, 0))
fi, err := NewFile("test", nd, dir, ds)
if err != nil {
t.Fatal(err)

2
mfs/system.go
View File

@ -83,7 +83,7 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFu
dserv: ds,
}
pbn, err := ft.FromBytes(node.Data)
pbn, err := ft.FromBytes(node.Data())
if err != nil {
log.Error("IPNS pointer was not unixfs node")
return nil, err

2
namesys/publisher.go
View File

@ -330,7 +330,7 @@ func ValidateIpnsRecord(k key.Key, val []byte) error {
// point to an empty directory.
// TODO: this doesnt feel like it belongs here
func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error {
emptyDir := &dag.Node{Data: ft.FolderPBData()}
emptyDir := ft.EmptyDirNode()
nodek, err := ds.Add(emptyDir)
if err != nil {
return err

4
path/resolver_test.go
View File

@ -15,8 +15,8 @@ import (
func randNode() (*merkledag.Node, key.Key) {
node := new(merkledag.Node)
node.Data = make([]byte, 32)
util.NewTimeSeededRand().Read(node.Data)
node.SetData(make([]byte, 32))
util.NewTimeSeededRand().Read(node.Data())
k, _ := node.Key()
return node, k
}

4
pin/pin_test.go
View File

@ -18,8 +18,8 @@ import (
func randNode() (*mdag.Node, key.Key) {
nd := new(mdag.Node)
nd.Data = make([]byte, 32)
util.NewTimeSeededRand().Read(nd.Data)
nd.SetData(make([]byte, 32))
util.NewTimeSeededRand().Read(nd.Data())
k, _ := nd.Key()
return nd, k
}

18
pin/set.go
View File

@ -111,7 +111,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
if err := writeHdr(n, hdr); err != nil {
return nil, err
}
hdrLen := len(n.Data)
hdrLen := len(n.Data())
if estimatedLen < maxItems {
// it'll probably fit
@ -122,12 +122,12 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
break
}
n.Links = append(n.Links, &merkledag.Link{Hash: k.ToMultihash()})
n.Data = append(n.Data, data...)
n.SetData(append(n.Data(), data...))
}
// sort by hash, also swap item Data
s := sortByHash{
links: n.Links[defaultFanout:],
data: n.Data[hdrLen:],
data: n.Data()[hdrLen:],
}
sort.Stable(s)
}
@ -179,11 +179,11 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
}
func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) {
hdrLenRaw, consumed := binary.Uvarint(n.Data)
hdrLenRaw, consumed := binary.Uvarint(n.Data())
if consumed <= 0 {
return nil, nil, errors.New("invalid Set header length")
}
buf := n.Data[consumed:]
buf := n.Data()[consumed:]
if hdrLenRaw > uint64(len(buf)) {
return nil, nil, errors.New("impossibly large Set header length")
}
@ -209,10 +209,10 @@ func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
if err != nil {
return err
}
n.Data = make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))
written := binary.PutUvarint(n.Data, uint64(len(hdrData)))
n.Data = n.Data[:written]
n.Data = append(n.Data, hdrData...)
n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)))
written := binary.PutUvarint(n.Data(), uint64(len(hdrData)))
n.SetData(n.Data()[:written])
n.SetData(append(n.Data(), hdrData...))
return nil
}

8
tar/format.go
View File

@ -45,7 +45,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
tr := tar.NewReader(r)
root := new(dag.Node)
root.Data = []byte("ipfs/tar")
root.SetData([]byte("ipfs/tar"))
e := dagutil.NewDagEditor(root, ds)
@ -65,7 +65,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
return nil, err
}
header.Data = headerBytes
header.SetData(headerBytes)
if h.Size > 0 {
spl := chunk.NewRabin(tr, uint64(chunk.DefaultBlockSize))
@ -170,7 +170,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
return 0, err
}
tr.hdrBuf = bytes.NewReader(headerNd.Data)
tr.hdrBuf = bytes.NewReader(headerNd.Data())
dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
if err != nil && err != dag.ErrLinkNotFound {
@ -197,7 +197,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
}
func ExportTar(ctx context.Context, root *dag.Node, ds dag.DAGService) (io.Reader, error) {
if string(root.Data) != "ipfs/tar" {
if string(root.Data()) != "ipfs/tar" {
return nil, errors.New("not an ipfs tarchive")
}
return &tarReader{

2
unixfs/archive/tar/writer.go
View File

@ -69,7 +69,7 @@ func (w *Writer) writeFile(nd *mdag.Node, pb *upb.Data, fpath string) error {
func (w *Writer) WriteNode(nd *mdag.Node, fpath string) error {
pb := new(upb.Data)
if err := proto.Unmarshal(nd.Data, pb); err != nil {
if err := proto.Unmarshal(nd.Data(), pb); err != nil {
return err
}

5
unixfs/format.go
View File

@ -6,6 +6,7 @@ package unixfs
import (
"errors"
dag "github.com/ipfs/go-ipfs/merkledag"
pb "github.com/ipfs/go-ipfs/unixfs/pb"
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
)
@ -222,3 +223,7 @@ func BytesForMetadata(m *Metadata) ([]byte, error) {
pbd.Data = mdd
return proto.Marshal(pbd)
}
func EmptyDirNode() *dag.Node {
return dag.NodeWithData(FolderPBData())
}

4
unixfs/io/dagreader.go
View File

@ -60,7 +60,7 @@ type ReadSeekCloser interface {
// the given node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
pb := new(ftpb.Data)
if err := proto.Unmarshal(n.Data, pb); err != nil {
if err := proto.Unmarshal(n.Data(), pb); err != nil {
return nil, err
}
@ -117,7 +117,7 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
dr.linkPosition++
pb := new(ftpb.Data)
err = proto.Unmarshal(nxt.Data, pb)
err = proto.Unmarshal(nxt.Data(), pb)
if err != nil {
return fmt.Errorf("incorrectly formatted protobuf: %s", err)
}

4
unixfs/io/dirbuilder.go
View File

@ -15,7 +15,9 @@ type directoryBuilder struct {
// NewEmptyDirectory returns an empty merkledag Node with a folder Data chunk
func NewEmptyDirectory() *mdag.Node {
return &mdag.Node{Data: format.FolderPBData()}
nd := new(mdag.Node)
nd.SetData(format.FolderPBData())
return nd
}
// NewDirectory returns a directoryBuilder. It needs a DAGService to add the Children

15
unixfs/mod/dagmodifier.go
View File

@ -139,7 +139,7 @@ func (dm *DagModifier) Write(b []byte) (int, error) {
}
func (dm *DagModifier) Size() (int64, error) {
pbn, err := ft.FromBytes(dm.curNode.Data)
pbn, err := ft.FromBytes(dm.curNode.Data())
if err != nil {
return 0, err
}
@ -207,7 +207,7 @@ func (dm *DagModifier) Sync() error {
// returns the new key of the passed in node and whether or not all the data in the reader
// has been consumed.
func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (key.Key, bool, error) {
f, err := ft.FromBytes(node.Data)
f, err := ft.FromBytes(node.Data())
if err != nil {
return "", false, err
}
@ -225,7 +225,8 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
return "", false, err
}
nd := &mdag.Node{Data: b}
nd := new(mdag.Node)
nd.SetData(b)
k, err := dm.dagserv.Add(nd)
if err != nil {
return "", false, err
@ -429,12 +430,12 @@ func (dm *DagModifier) Truncate(size int64) error {
func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, error) {
if len(nd.Links) == 0 {
// TODO: this can likely be done without marshaling and remarshaling
pbn, err := ft.FromBytes(nd.Data)
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return nil, err
}
nd.Data = ft.WrapData(pbn.Data[:size])
nd.SetData(ft.WrapData(pbn.Data[:size]))
return nd, nil
}
@ -448,7 +449,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer
return nil, err
}
childsize, err := ft.DataSize(child.Data)
childsize, err := ft.DataSize(child.Data())
if err != nil {
return nil, err
}
@ -486,7 +487,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer
return nil, err
}
nd.Data = d
nd.SetData(d)
// invalidate cache and recompute serialized data
_, err = nd.EncodeProtobuf(true)

4
unixfs/mod/dagmodifier_test.go
View File

@ -156,7 +156,7 @@ func TestDagModifierBasic(t *testing.T) {
t.Fatal(err)
}
size, err := ft.DataSize(node.Data)
size, err := ft.DataSize(node.Data())
if err != nil {
t.Fatal(err)
}
@ -590,7 +590,7 @@ func arrComp(a, b []byte) error {
}
func printDag(nd *mdag.Node, ds mdag.DAGService, indent int) {
pbd, err := ft.FromBytes(nd.Data)
pbd, err := ft.FromBytes(nd.Data())
if err != nil {
panic(err)
}