mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-23 19:37:46 +08:00
This commit adds a new set of sharness tests for pinning, and addresses bugs that were pointed out by said tests. test/sharness: added more pinning tests Pinning is currently broken. See issue #1051. This commit introduces a few more pinning tests. These are by no means exhaustive, but definitely surface the present problems going on. I believe these tests are correct, but not sure. Pushing them as failing so that pinning is fixed in this PR. make pinning and merkledag.Get take contexts improve 'add' commands usage of pinning FIXUP: fix 'pin lists look good' ipfs-pin-stat simple script to help check pinning This is a simple shell script to help check pinning. We ought to strive towards making adding commands this easy. The http api is great and powerful, but our setup right now gets in the way. Perhaps we can clean up that area. updated t0081-repo-pinning - fixed a couple bugs with the tests - made it a bit clearer (still a lot going on) - the remaining tests are correct and highlight a problem with pinning. Namely, that recursive pinning is buggy. At least: towards the end of the test, $HASH_DIR4 and $HASH_FILE4 should be pinned indirectly, but they're not. And thus get gc-ed out. There may be other problems too. cc @whyrusleeping fix grep params for context deadline check fix bugs in pin and pin tests check for block local before checking recursive pin
234 lines
5.2 KiB
Go
234 lines
5.2 KiB
Go
package merkledag
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
|
|
|
|
mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
|
|
u "github.com/ipfs/go-ipfs/util"
|
|
)
|
|
|
|
// NodeMap maps u.Keys to Nodes.
|
|
// We cannot use []byte/Multihash for keys :(
|
|
// so have to convert Multihash bytes to string (u.Key)
|
|
type NodeMap map[u.Key]*Node
|
|
|
|
// Node represents a node in the IPFS Merkle DAG.
|
|
// nodes have opaque data and a set of navigable links.
|
|
type Node struct {
|
|
Links []*Link
|
|
Data []byte
|
|
|
|
// cache encoded/marshaled value
|
|
encoded []byte
|
|
|
|
cached mh.Multihash
|
|
}
|
|
|
|
// NodeStat is a statistics object for a Node. Mostly sizes.
|
|
type NodeStat struct {
|
|
NumLinks int // number of links in link table
|
|
BlockSize int // size of the raw, encoded data
|
|
LinksSize int // size of the links segment
|
|
DataSize int // size of the data segment
|
|
CumulativeSize int // cumulative size of object and its references
|
|
}
|
|
|
|
func (ns NodeStat) String() string {
|
|
f := "NodeStat{NumLinks: %d, BlockSize: %d, LinksSize: %d, DataSize: %d, CumulativeSize: %d}"
|
|
return fmt.Sprintf(f, ns.NumLinks, ns.BlockSize, ns.LinksSize, ns.DataSize, ns.CumulativeSize)
|
|
}
|
|
|
|
// Link represents an IPFS Merkle DAG Link between Nodes.
|
|
type Link struct {
|
|
// utf string name. should be unique per object
|
|
Name string // utf8
|
|
|
|
// cumulative size of target object
|
|
Size uint64
|
|
|
|
// multihash of the target object
|
|
Hash mh.Multihash
|
|
|
|
// a ptr to the actual node for graph manipulation
|
|
Node *Node
|
|
}
|
|
|
|
type LinkSlice []*Link
|
|
|
|
func (ls LinkSlice) Len() int { return len(ls) }
|
|
func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] }
|
|
func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name }
|
|
|
|
// MakeLink creates a link to the given node
|
|
func MakeLink(n *Node) (*Link, error) {
|
|
s, err := n.Size()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
h, err := n.Multihash()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Link{
|
|
Size: s,
|
|
Hash: h,
|
|
}, nil
|
|
}
|
|
|
|
// GetNode returns the MDAG Node that this link points to
|
|
func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
|
|
if l.Node != nil {
|
|
return l.Node, nil
|
|
}
|
|
|
|
return serv.Get(ctx, u.Key(l.Hash))
|
|
}
|
|
|
|
// AddNodeLink adds a link to another node.
|
|
func (n *Node) AddNodeLink(name string, that *Node) error {
|
|
n.encoded = nil
|
|
|
|
lnk, err := MakeLink(that)
|
|
|
|
lnk.Name = name
|
|
lnk.Node = that
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
n.AddRawLink(name, lnk)
|
|
|
|
return nil
|
|
}
|
|
|
|
// AddNodeLinkClean adds a link to another node. without keeping a reference to
|
|
// the child node
|
|
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
|
|
n.encoded = nil
|
|
lnk, err := MakeLink(that)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n.AddRawLink(name, lnk)
|
|
|
|
return nil
|
|
}
|
|
|
|
// AddRawLink adds a copy of a link to this node
|
|
func (n *Node) AddRawLink(name string, l *Link) error {
|
|
n.encoded = nil
|
|
n.Links = append(n.Links, &Link{
|
|
Name: name,
|
|
Size: l.Size,
|
|
Hash: l.Hash,
|
|
Node: l.Node,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Remove a link on this node by the given name
|
|
func (n *Node) RemoveNodeLink(name string) error {
|
|
n.encoded = nil
|
|
for i, l := range n.Links {
|
|
if l.Name == name {
|
|
n.Links = append(n.Links[:i], n.Links[i+1:]...)
|
|
return nil
|
|
}
|
|
}
|
|
return ErrNotFound
|
|
}
|
|
|
|
// Return a copy of the link with given name
|
|
func (n *Node) GetNodeLink(name string) (*Link, error) {
|
|
for _, l := range n.Links {
|
|
if l.Name == name {
|
|
return &Link{
|
|
Name: l.Name,
|
|
Size: l.Size,
|
|
Hash: l.Hash,
|
|
Node: l.Node,
|
|
}, nil
|
|
}
|
|
}
|
|
return nil, ErrNotFound
|
|
}
|
|
|
|
// Copy returns a copy of the node.
|
|
// NOTE: does not make copies of Node objects in the links.
|
|
func (n *Node) Copy() *Node {
|
|
nnode := new(Node)
|
|
nnode.Data = make([]byte, len(n.Data))
|
|
copy(nnode.Data, n.Data)
|
|
|
|
nnode.Links = make([]*Link, len(n.Links))
|
|
copy(nnode.Links, n.Links)
|
|
return nnode
|
|
}
|
|
|
|
// UpdateNodeLink return a copy of the node with the link name set to point to
|
|
// that. If a link of the same name existed, it is removed.
|
|
func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) {
|
|
newnode := n.Copy()
|
|
err := newnode.RemoveNodeLink(name)
|
|
err = nil // ignore error
|
|
err = newnode.AddNodeLink(name, that)
|
|
return newnode, err
|
|
}
|
|
|
|
// Size returns the total size of the data addressed by node,
|
|
// including the total sizes of references.
|
|
func (n *Node) Size() (uint64, error) {
|
|
b, err := n.Encoded(false)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
s := uint64(len(b))
|
|
for _, l := range n.Links {
|
|
s += l.Size
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// Stat returns statistics on the node.
|
|
func (n *Node) Stat() (*NodeStat, error) {
|
|
enc, err := n.Encoded(false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cumSize, err := n.Size()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &NodeStat{
|
|
NumLinks: len(n.Links),
|
|
BlockSize: len(enc),
|
|
LinksSize: len(enc) - len(n.Data), // includes framing.
|
|
DataSize: len(n.Data),
|
|
CumulativeSize: int(cumSize),
|
|
}, nil
|
|
}
|
|
|
|
// Multihash hashes the encoded data of this node.
|
|
func (n *Node) Multihash() (mh.Multihash, error) {
|
|
// Note: Encoded generates the hash and puts it in n.cached.
|
|
_, err := n.Encoded(false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return n.cached, nil
|
|
}
|
|
|
|
// Key returns the Multihash as a key, for maps.
|
|
func (n *Node) Key() (u.Key, error) {
|
|
h, err := n.Multihash()
|
|
return u.Key(h), err
|
|
}
|