From 6458ddcd32291b4e1fef79c7b5d2e0c080fc0cfe Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 19 Oct 2014 15:28:38 -0700 Subject: [PATCH] flesh out pinning object, needs tests and cli wiring still --- Godeps/Godeps.json | 2 +- .../jbenet/datastore.go/keytransform/doc.go | 25 +++++++ .../datastore.go/keytransform/interface.go | 34 +++++++++ .../jbenet/datastore.go/namespace/doc.go | 24 +++++++ .../datastore.go/namespace/example_test.go | 30 ++++++++ .../datastore.go/namespace/namespace.go | 44 ++++++++++++ .../datastore.go/namespace/namespace_test.go | 72 +++++++++++++++++++ .../jbenet/go-datastore/fs/fs_test.go | 22 ++++-- .../github.com/jbenet/go-datastore/io/io.go | 44 ------------ blocks/set/dbset.go | 12 ++-- blocks/set/set.go | 35 +++++++++ pin/indirect.go | 34 ++++++--- pin/pin.go | 62 ++++++++++++++-- 13 files changed, 370 insertions(+), 70 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/doc.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/interface.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/doc.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/example_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace_test.go delete mode 100644 Godeps/_workspace/src/github.com/jbenet/go-datastore/io/io.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 0a2b0780b..51e93adfa 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/jbenet/go-ipfs", - "GoVersion": "go1.3", + "GoVersion": "go1.3.3", "Packages": [ "./..." ], diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/doc.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/doc.go new file mode 100644 index 000000000..208b7d114 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/doc.go @@ -0,0 +1,25 @@ +// Package keytransform introduces a Datastore Shim that transforms keys before +// passing them to its child. It can be used to manipulate what keys look like +// to the user, for example namespacing keys, reversing them, etc. +// +// Use the Wrap function to wrap a datastore with any KeyTransform. +// A KeyTransform is simply an interface with two functions, a conversion and +// its inverse. For example: +// +// import ( +// ktds "github.com/jbenet/datastore.go/keytransform" +// ds "github.com/jbenet/datastore.go" +// ) +// +// func reverseKey(k ds.Key) ds.Key { +// return k.Reverse() +// } +// +// func invertKeys(d ds.Datastore) { +// return ktds.Wrap(d, &ktds.Pair{ +// Convert: reverseKey, +// Invert: reverseKey, // reverse is its own inverse. +// }) +// } +// +package keytransform diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/interface.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/interface.go new file mode 100644 index 000000000..3ae83f357 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/interface.go @@ -0,0 +1,34 @@ +package keytransform + +import ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + +// KeyMapping is a function that maps one key to annother +type KeyMapping func(ds.Key) ds.Key + +// KeyTransform is an object with a pair of functions for (invertibly) +// transforming keys +type KeyTransform interface { + ConvertKey(ds.Key) ds.Key + InvertKey(ds.Key) ds.Key +} + +// Datastore is a keytransform.Datastore +type Datastore interface { + ds.Shim + KeyTransform +} + +// Wrap wraps a given datastore with a KeyTransform function. +// The resulting wrapped datastore will use the transform on all Datastore +// operations. +func Wrap(child ds.Datastore, t KeyTransform) Datastore { + if t == nil { + panic("t (KeyTransform) is nil") + } + + if child == nil { + panic("child (ds.Datastore) is nil") + } + + return &ktds{child: child, KeyTransform: t} +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/doc.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/doc.go new file mode 100644 index 000000000..2d7dcad4d --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/doc.go @@ -0,0 +1,24 @@ +// Package namespace introduces a namespace Datastore Shim, which basically +// mounts the entire child datastore under a prefix. +// +// Use the Wrap function to wrap a datastore with any Key prefix. For example: +// +// import ( +// "fmt" +// +// ds "github.com/jbenet/datastore.go" +// nsds "github.com/jbenet/datastore.go/namespace" +// ) +// +// func main() { +// mp := ds.NewMapDatastore() +// ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) +// +// // in the Namespace Datastore: +// ns.Put(ds.NewKey("/beep"), "boop") +// v2, _ := ns.Get(ds.NewKey("/beep")) // v2 == "boop" +// +// // and, in the underlying MapDatastore: +// v3, _ := mp.Get(ds.NewKey("/foo/bar/beep")) // v3 == "boop" +// } +package namespace diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/example_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/example_test.go new file mode 100644 index 000000000..f42caf3b6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/example_test.go @@ -0,0 +1,30 @@ +package namespace_test + +import ( + "fmt" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + nsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace" +) + +func Example() { + mp := ds.NewMapDatastore() + ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) + + k := ds.NewKey("/beep") + v := "boop" + + ns.Put(k, v) + fmt.Printf("ns.Put %s %s\n", k, v) + + v2, _ := ns.Get(k) + fmt.Printf("ns.Get %s -> %s\n", k, v2) + + k3 := ds.NewKey("/foo/bar/beep") + v3, _ := mp.Get(k3) + fmt.Printf("mp.Get %s -> %s\n", k3, v3) + // Output: + // ns.Put /beep -> boop + // ns.Get /beep -> boop + // mp.Get /foo/bar/beep -> boop +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace.go new file mode 100644 index 000000000..815385f89 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace.go @@ -0,0 +1,44 @@ +package namespace + +import ( + "fmt" + "strings" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + ktds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform" +) + +// PrefixTransform constructs a KeyTransform with a pair of functions that +// add or remove the given prefix key. +// +// Warning: will panic if prefix not found when it should be there. This is +// to avoid insidious data inconsistency errors. +func PrefixTransform(prefix ds.Key) ktds.KeyTransform { + return &ktds.Pair{ + + // Convert adds the prefix + Convert: func(k ds.Key) ds.Key { + return prefix.Child(k) + }, + + // Invert removes the prefix. panics if prefix not found. + Invert: func(k ds.Key) ds.Key { + if !prefix.IsAncestorOf(k) { + fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k) + panic("expected prefix not found") + } + + s := strings.TrimPrefix(k.String(), prefix.String()) + return ds.NewKey(s) + }, + } +} + +// Wrap wraps a given datastore with a key-prefix. +func Wrap(child ds.Datastore, prefix ds.Key) ktds.Datastore { + if child == nil { + panic("child (ds.Datastore) is nil") + } + + return ktds.Wrap(child, PrefixTransform(prefix)) +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace_test.go new file mode 100644 index 000000000..f49a5f9da --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace_test.go @@ -0,0 +1,72 @@ +package namespace_test + +import ( + "bytes" + "sort" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + ns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct{} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) TestBasic(c *C) { + + mpds := ds.NewMapDatastore() + nsds := ns.Wrap(mpds, ds.NewKey("abc")) + + keys := strsToKeys([]string{ + "foo", + "foo/bar", + "foo/bar/baz", + "foo/barb", + "foo/bar/bazb", + "foo/bar/baz/barb", + }) + + for _, k := range keys { + err := nsds.Put(k, []byte(k.String())) + c.Check(err, Equals, nil) + } + + for _, k := range keys { + v1, err := nsds.Get(k) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true) + + v2, err := mpds.Get(ds.NewKey("abc").Child(k)) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true) + } + + listA, errA := mpds.KeyList() + listB, errB := nsds.KeyList() + c.Check(errA, Equals, nil) + c.Check(errB, Equals, nil) + c.Check(len(listA), Equals, len(listB)) + + // sort them cause yeah. + sort.Sort(ds.KeySlice(listA)) + sort.Sort(ds.KeySlice(listB)) + + for i, kA := range listA { + kB := listB[i] + c.Check(nsds.InvertKey(kA), Equals, kB) + c.Check(kA, Equals, nsds.ConvertKey(kB)) + } +} + +func strsToKeys(strs []string) []ds.Key { + keys := make([]ds.Key, len(strs)) + for i, s := range strs { + keys[i] = ds.NewKey(s) + } + return keys +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go index 8e5d94ac4..9c41d3527 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go @@ -2,6 +2,7 @@ package fs_test import ( "bytes" + "sort" "testing" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" @@ -12,10 +13,7 @@ import ( // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { TestingT(t) } -type DSSuite struct { - dir string - ds ds.Datastore -} +type DSSuite struct{} var _ = Suite(&DSSuite{}) @@ -54,6 +52,22 @@ func (ks *DSSuite) TestBasic(c *C) { c.Check(err, Equals, nil) c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true) } + + listA, errA := mpds.KeyList() + listB, errB := ktds.KeyList() + c.Check(errA, Equals, nil) + c.Check(errB, Equals, nil) + c.Check(len(listA), Equals, len(listB)) + + // sort them cause yeah. + sort.Sort(ds.KeySlice(listA)) + sort.Sort(ds.KeySlice(listB)) + + for i, kA := range listA { + kB := listB[i] + c.Check(pair.Invert(kA), Equals, kB) + c.Check(kA, Equals, pair.Convert(kB)) + } } func strsToKeys(strs []string) []ds.Key { diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/io/io.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/io/io.go deleted file mode 100644 index 3d5d2c770..000000000 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/io/io.go +++ /dev/null @@ -1,44 +0,0 @@ -package leveldb - -import ( - "bytes" - "io" - - ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" -) - -// CastAsReader does type assertions to find the type of a value and attempts -// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType -func CastAsReader(value interface{}) (io.Reader, error) { - switch v := value.(type) { - case io.Reader: - return v, nil - - case []byte: - return bytes.NewReader(v), nil - - case string: - return bytes.NewReader([]byte(v)), nil - - default: - return nil, ds.ErrInvalidType - } -} - -// // CastAsWriter does type assertions to find the type of a value and attempts -// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType -// func CastAsWriter(value interface{}) (err error) { -// switch v := value.(type) { -// case io.Reader: -// return v, nil -// -// case []byte: -// return bytes.NewReader(v), nil -// -// case string: -// return bytes.NewReader([]byte(v)), nil -// -// default: -// return nil, ds.ErrInvalidType -// } -// } diff --git a/blocks/set/dbset.go b/blocks/set/dbset.go index 02e32be2b..187ca8ac6 100644 --- a/blocks/set/dbset.go +++ b/blocks/set/dbset.go @@ -9,19 +9,17 @@ import ( type datastoreBlockSet struct { dstore ds.Datastore bset BlockSet - prefix string } -func NewDBWrapperSet(d ds.Datastore, prefix string, bset BlockSet) BlockSet { +func NewDBWrapperSet(d ds.Datastore, bset BlockSet) BlockSet { return &datastoreBlockSet{ dstore: d, bset: bset, - prefix: prefix, } } func (d *datastoreBlockSet) AddBlock(k util.Key) { - err := d.dstore.Put(d.prefixKey(k), []byte{}) + err := d.dstore.Put(k.DsKey(), []byte{}) if err != nil { log.Error("blockset put error: %s", err) } @@ -32,7 +30,7 @@ func (d *datastoreBlockSet) AddBlock(k util.Key) { func (d *datastoreBlockSet) RemoveBlock(k util.Key) { d.bset.RemoveBlock(k) if !d.bset.HasKey(k) { - d.dstore.Delete(d.prefixKey(k)) + d.dstore.Delete(k.DsKey()) } } @@ -44,6 +42,6 @@ func (d *datastoreBlockSet) GetBloomFilter() bloom.Filter { return d.bset.GetBloomFilter() } -func (d *datastoreBlockSet) prefixKey(k util.Key) ds.Key { - return (util.Key(d.prefix) + k).DsKey() +func (d *datastoreBlockSet) GetKeys() []util.Key { + return d.bset.GetKeys() } diff --git a/blocks/set/set.go b/blocks/set/set.go index 7d0ee23fe..7288e26c3 100644 --- a/blocks/set/set.go +++ b/blocks/set/set.go @@ -1,6 +1,10 @@ package set import ( + "errors" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + "github.com/jbenet/go-ipfs/blocks/bloom" "github.com/jbenet/go-ipfs/util" ) @@ -12,6 +16,29 @@ type BlockSet interface { RemoveBlock(util.Key) HasKey(util.Key) bool GetBloomFilter() bloom.Filter + + GetKeys() []util.Key +} + +func SimpleSetFromKeys(keys []util.Key) BlockSet { + sbs := &simpleBlockSet{blocks: make(map[util.Key]struct{})} + for _, k := range keys { + sbs.blocks[k] = struct{}{} + } + return sbs +} + +func SetFromDatastore(d ds.Datastore, k ds.Key) (BlockSet, error) { + ikeys, err := d.Get(k) + if err != nil { + return nil, err + } + + keys, ok := ikeys.([]util.Key) + if !ok { + return nil, errors.New("Incorrect type for keys from datastore") + } + return SimpleSetFromKeys(keys), nil } func NewSimpleBlockSet() BlockSet { @@ -42,3 +69,11 @@ func (b *simpleBlockSet) GetBloomFilter() bloom.Filter { } return f } + +func (b *simpleBlockSet) GetKeys() []util.Key { + var out []util.Key + for k, _ := range b.blocks { + out = append(out, k) + } + return out +} diff --git a/pin/indirect.go b/pin/indirect.go index b3b60e753..bff1e545f 100644 --- a/pin/indirect.go +++ b/pin/indirect.go @@ -1,25 +1,41 @@ package pin import ( + "errors" + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - bc "github.com/jbenet/go-ipfs/blocks/set" + "github.com/jbenet/go-ipfs/blocks/set" "github.com/jbenet/go-ipfs/util" ) type indirectPin struct { - blockset bc.BlockSet + blockset set.BlockSet refCounts map[util.Key]int } -func loadBlockSet(d ds.Datastore) (bc.BlockSet, map[util.Key]int) { - panic("Not yet implemented!") - return nil, nil +func NewIndirectPin(dstore ds.Datastore) *indirectPin { + return &indirectPin{ + blockset: set.NewDBWrapperSet(dstore, set.NewSimpleBlockSet()), + refCounts: make(map[util.Key]int), + } } -func newIndirectPin(d ds.Datastore) indirectPin { - // suppose the blockset actually takes blocks, not just keys - bs, rc := loadBlockSet(d) - return indirectPin{bs, rc} +func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) { + irefcnt, err := d.Get(k) + if err != nil { + return nil, err + } + refcnt, ok := irefcnt.(map[util.Key]int) + if !ok { + return nil, errors.New("invalid type from datastore") + } + + var keys []util.Key + for k, _ := range refcnt { + keys = append(keys, k) + } + + return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil } func (i *indirectPin) Increment(k util.Key) { diff --git a/pin/pin.go b/pin/pin.go index 86fff2afb..d6f7f0f19 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -1,21 +1,29 @@ package pin import ( + + //ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + nsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace" "github.com/jbenet/go-ipfs/blocks/set" mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/util" ) +var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys") +var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys") +var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys") + type Pinner interface { Pin(*mdag.Node, bool) error Unpin(util.Key, bool) error + Flush() error } type pinner struct { recursePin set.BlockSet directPin set.BlockSet - indirPin indirectPin + indirPin *indirectPin dserv *mdag.DAGService dstore ds.Datastore } @@ -23,14 +31,17 @@ type pinner struct { func NewPinner(dstore ds.Datastore, serv *mdag.DAGService) Pinner { // Load set from given datastore... - rcset := set.NewDBWrapperSet(dstore, "/pinned/recurse/", set.NewSimpleBlockSet()) - dirset := set.NewDBWrapperSet(dstore, "/pinned/direct/", set.NewSimpleBlockSet()) + rcds := nsds.Wrap(dstore, recursePinDatastoreKey) + rcset := set.NewDBWrapperSet(rcds, set.NewSimpleBlockSet()) - nsdstore := dstore // WRAP IN NAMESPACE + dirds := nsds.Wrap(dstore, directPinDatastoreKey) + dirset := set.NewDBWrapperSet(dirds, set.NewSimpleBlockSet()) + + nsdstore := nsds.Wrap(dstore, indirectPinDatastoreKey) return &pinner{ recursePin: rcset, directPin: dirset, - indirPin: newIndirectPin(nsdstore), + indirPin: NewIndirectPin(nsdstore), dserv: serv, dstore: dstore, } @@ -126,3 +137,44 @@ func (p *pinner) IsPinned(key util.Key) bool { p.directPin.HasKey(key) || p.indirPin.HasKey(key) } + +func LoadPinner(d ds.Datastore) (Pinner, error) { + p := new(pinner) + + var err error + p.recursePin, err = set.SetFromDatastore(d, recursePinDatastoreKey) + if err != nil { + return nil, err + } + p.directPin, err = set.SetFromDatastore(d, directPinDatastoreKey) + if err != nil { + return nil, err + } + + p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey) + if err != nil { + return nil, err + } + + return p, nil +} + +func (p *pinner) Flush() error { + recurse := p.recursePin.GetKeys() + err := p.dstore.Put(recursePinDatastoreKey, recurse) + if err != nil { + return err + } + + direct := p.directPin.GetKeys() + err = p.dstore.Put(directPinDatastoreKey, direct) + if err != nil { + return err + } + + err = p.dstore.Put(indirectPinDatastoreKey, p.indirPin.refCounts) + if err != nil { + return err + } + return nil +}