From 8a7f6aca99cf6018bff6b0a20b8ad14a59d27bee Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:38:02 -0700 Subject: [PATCH 1/4] godeps: updated datastore --- Godeps/Godeps.json | 2 +- .../jbenet/datastore.go/Godeps/Godeps.json | 43 ++++++ .../jbenet/datastore.go/Godeps/Readme | 5 + .../jbenet/datastore.go/basic_ds.go | 65 +++++++--- .../jbenet/datastore.go/basic_ds_test.go | 13 ++ .../jbenet/datastore.go/datastore.go | 16 ++- .../github.com/jbenet/datastore.go/fs/fs.go | 122 ++++++++++++++++++ .../jbenet/datastore.go/fs/fs_test.go | 65 ++++++++++ .../github.com/jbenet/datastore.go/io/io.go | 44 +++++++ .../jbenet/datastore.go/key_test.go | 5 +- .../jbenet/datastore.go/leveldb/datastore.go | 4 +- .../jbenet/datastore.go/lru/datastore.go | 54 ++++++++ .../jbenet/datastore.go/lru/datastore_test.go | 52 ++++++++ .../github.com/jbenet/datastore.go/query.go | 19 +++ .../jbenet/datastore.go/sync/sync.go | 64 +++++++++ 15 files changed, 546 insertions(+), 27 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index fbe7494b0..4e5dfdf27 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -64,7 +64,7 @@ }, { "ImportPath": "github.com/jbenet/datastore.go", - "Rev": "e89f0511689bb2d0608496e15491f241842de085" + "Rev": "e7d6f7cb9e3c207a04c5397c449d10a6f9d403a0" }, { "ImportPath": "github.com/jbenet/go-base58", diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json new file mode 100644 index 000000000..bebf4a58f --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json @@ -0,0 +1,43 @@ +{ + "ImportPath": "github.com/jbenet/datastore.go", + "GoVersion": "go1.3.1", + "Packages": [ + "./..." + ], + "Deps": [ + { + "ImportPath": "code.google.com/p/go-uuid/uuid", + "Comment": "null-12", + "Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9" + }, + { + "ImportPath": "code.google.com/p/snappy-go/snappy", + "Comment": "null-15", + "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" + }, + { + "ImportPath": "github.com/codahale/blake2", + "Rev": "3fa823583afba430e8fc7cdbcc670dbf90bfacc4" + }, + { + "ImportPath": "github.com/hashicorp/golang-lru", + "Rev": "4dfff096c4973178c8f35cf6dd1a732a0a139370" + }, + { + "ImportPath": "github.com/mattbaird/elastigo/api", + "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d" + }, + { + "ImportPath": "github.com/mattbaird/elastigo/core", + "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d" + }, + { + "ImportPath": "github.com/syndtr/goleveldb/leveldb", + "Rev": "9bca75c48d6c31becfbb127702b425e7226052e3" + }, + { + "ImportPath": "gopkg.in/check.v1", + "Rev": "91ae5f88a67b14891cfd43895b01164f6c120420" + } + ] +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme new file mode 100644 index 000000000..4cdaa53d5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go index 33ea5f3a7..782b5de46 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go @@ -1,28 +1,30 @@ package datastore -import ( - "log" -) +import "log" // Here are some basic datastore implementations. -// MapDatastore uses a standard Go map for internal storage. type keyMap map[Key]interface{} + +// MapDatastore uses a standard Go map for internal storage. type MapDatastore struct { values keyMap } +// NewMapDatastore constructs a MapDatastore func NewMapDatastore() (d *MapDatastore) { return &MapDatastore{ values: keyMap{}, } } +// Put implements Datastore.Put func (d *MapDatastore) Put(key Key, value interface{}) (err error) { d.values[key] = value return nil } +// Get implements Datastore.Get func (d *MapDatastore) Get(key Key) (value interface{}, err error) { val, found := d.values[key] if !found { @@ -31,19 +33,22 @@ func (d *MapDatastore) Get(key Key) (value interface{}, err error) { return val, nil } +// Has implements Datastore.Has func (d *MapDatastore) Has(key Key) (exists bool, err error) { _, found := d.values[key] return found, nil } +// Delete implements Datastore.Delete func (d *MapDatastore) Delete(key Key) (err error) { delete(d.values, key) return nil } +// KeyList implements Datastore.KeyList func (d *MapDatastore) KeyList() ([]Key, error) { var keys []Key - for k, _ := range d.values { + for k := range d.values { keys = append(keys, k) } return keys, nil @@ -54,26 +59,32 @@ func (d *MapDatastore) KeyList() ([]Key, error) { type NullDatastore struct { } +// NewNullDatastore constructs a null datastoe func NewNullDatastore() *NullDatastore { return &NullDatastore{} } +// Put implements Datastore.Put func (d *NullDatastore) Put(key Key, value interface{}) (err error) { return nil } +// Get implements Datastore.Get func (d *NullDatastore) Get(key Key) (value interface{}, err error) { return nil, nil } +// Has implements Datastore.Has func (d *NullDatastore) Has(key Key) (exists bool, err error) { return false, nil } +// Delete implements Datastore.Delete func (d *NullDatastore) Delete(key Key) (err error) { return nil } +// KeyList implements Datastore.KeyList func (d *NullDatastore) KeyList() ([]Key, error) { return nil, nil } @@ -81,38 +92,56 @@ func (d *NullDatastore) KeyList() ([]Key, error) { // LogDatastore logs all accesses through the datastore. type LogDatastore struct { Name string - Child Datastore + child Datastore } -func NewLogDatastore(ds Datastore, name string) *LogDatastore { +// Shim is a datastore which has a child. +type Shim interface { + Datastore + + Children() []Datastore +} + +// NewLogDatastore constructs a log datastore. +func NewLogDatastore(ds Datastore, name string) Shim { if len(name) < 1 { name = "LogDatastore" } - return &LogDatastore{Name: name, Child: ds} + return &LogDatastore{Name: name, child: ds} } +// Children implements Shim +func (d *LogDatastore) Children() []Datastore { + return []Datastore{d.child} +} + +// Put implements Datastore.Put func (d *LogDatastore) Put(key Key, value interface{}) (err error) { - log.Printf("%s: Put %s", d.Name, key) + log.Printf("%s: Put %s\n", d.Name, key) // log.Printf("%s: Put %s ```%s```", d.Name, key, value) - return d.Child.Put(key, value) + return d.child.Put(key, value) } +// Get implements Datastore.Get func (d *LogDatastore) Get(key Key) (value interface{}, err error) { - log.Printf("%s: Get %s", d.Name, key) - return d.Child.Get(key) + log.Printf("%s: Get %s\n", d.Name, key) + return d.child.Get(key) } +// Has implements Datastore.Has func (d *LogDatastore) Has(key Key) (exists bool, err error) { - log.Printf("%s: Has %s", d.Name, key) - return d.Child.Has(key) + log.Printf("%s: Has %s\n", d.Name, key) + return d.child.Has(key) } +// Delete implements Datastore.Delete func (d *LogDatastore) Delete(key Key) (err error) { - log.Printf("%s: Delete %s", d.Name, key) - return d.Child.Delete(key) + log.Printf("%s: Delete %s\n", d.Name, key) + return d.child.Delete(key) } +// KeyList implements Datastore.KeyList func (d *LogDatastore) KeyList() ([]Key, error) { - log.Printf("%s: Get KeyList.", d.Name) - return d.Child.KeyList() + log.Printf("%s: Get KeyList\n", d.Name) + return d.child.KeyList() } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go new file mode 100644 index 000000000..e175d94da --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go @@ -0,0 +1,13 @@ +package datastore_test + +import ( + . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type BasicSuite struct{} + +var _ = Suite(&BasicSuite{}) diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go index 9ff21a6a0..f3260cd7d 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go @@ -5,7 +5,7 @@ import ( ) /* -A Datastore represents storage for any key-value pair. +Datastore represents storage for any key-value pair. Datastores are general enough to be backed by all kinds of different storage: in-memory caches, databases, a remote datastore, flat files on disk, etc. @@ -27,7 +27,6 @@ and thus it should behave predictably and handle exceptional conditions with proper error reporting. Thus, all Datastore calls may return errors, which should be checked by callers. */ - type Datastore interface { // Put stores the object `value` named by `key`. // @@ -53,20 +52,27 @@ type Datastore interface { // Delete removes the value for given `key`. Delete(key Key) (err error) - // Returns a list of keys in the datastore + // KeyList returns a list of keys in the datastore KeyList() ([]Key, error) } +// ThreadSafeDatastore is an interface that all threadsafe datastore should +// implement to leverage type safety checks. +type ThreadSafeDatastore interface { + Datastore + IsThreadSafe() +} + // Errors // ErrNotFound is returned by Get, Has, and Delete when a datastore does not // map the given key to a value. -var ErrNotFound = errors.New("datastore: key not found.") +var ErrNotFound = errors.New("datastore: key not found") // ErrInvalidType is returned by Put when a given value is incopatible with // the type the datastore supports. This means a conversion (or serialization) // is needed beforehand. -var ErrInvalidType = errors.New("datastore: invalid type error.") +var ErrInvalidType = errors.New("datastore: invalid type error") // GetBackedHas provides a default Datastore.Has implementation. // It exists so Datastore.Has implementations can use it, like so: diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go new file mode 100644 index 000000000..1c1ac791d --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go @@ -0,0 +1,122 @@ +package fs + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// Datastore uses a standard Go map for internal storage. +type Datastore struct { + path string +} + +// NewDatastore returns a new fs Datastore at given `path` +func NewDatastore(path string) (ds.Datastore, error) { + if !isDir(path) { + return nil, fmt.Errorf("Failed to find directory at: %v (file? perms?)", path) + } + + return &Datastore{path: path}, nil +} + +// KeyFilename returns the filename associated with `key` +func (d *Datastore) KeyFilename(key ds.Key) string { + return filepath.Join(d.path, key.String(), ".dsobject") +} + +// Put stores the given value. +func (d *Datastore) Put(key ds.Key, value interface{}) (err error) { + + // TODO: maybe use io.Readers/Writers? + // r, err := dsio.CastAsReader(value) + // if err != nil { + // return err + // } + + val, ok := value.([]byte) + if !ok { + return ds.ErrInvalidType + } + + fn := d.KeyFilename(key) + + // mkdirall above. + err = os.MkdirAll(filepath.Dir(fn), 0755) + if err != nil { + return err + } + + return ioutil.WriteFile(fn, val, 0666) +} + +// Get returns the value for given key +func (d *Datastore) Get(key ds.Key) (value interface{}, err error) { + fn := d.KeyFilename(key) + if !isFile(fn) { + return nil, ds.ErrNotFound + } + + return ioutil.ReadFile(fn) +} + +// Has returns whether the datastore has a value for a given key +func (d *Datastore) Has(key ds.Key) (exists bool, err error) { + return ds.GetBackedHas(d, key) +} + +// Delete removes the value for given key +func (d *Datastore) Delete(key ds.Key) (err error) { + fn := d.KeyFilename(key) + if !isFile(fn) { + return ds.ErrNotFound + } + + return os.Remove(fn) +} + +// KeyList returns a list of all keys in the datastore +func (d *Datastore) KeyList() ([]ds.Key, error) { + + keys := []ds.Key{} + + walkFn := func(path string, info os.FileInfo, err error) error { + // remove ds path prefix + if strings.HasPrefix(path, d.path) { + path = path[len(d.path):] + } + + if !info.IsDir() { + key := ds.NewKey(path) + keys = append(keys, key) + } + return nil + } + + filepath.Walk(d.path, walkFn) + return keys, nil +} + +// isDir returns whether given path is a directory +func isDir(path string) bool { + finfo, err := os.Stat(path) + if err != nil { + return false + } + + return finfo.IsDir() +} + +// isFile returns whether given path is a file +func isFile(path string) bool { + finfo, err := os.Stat(path) + if err != nil { + return false + } + + return !finfo.IsDir() +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go new file mode 100644 index 000000000..cd69647d0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go @@ -0,0 +1,65 @@ +package fs_test + +import ( + "bytes" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + fs "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct { + dir string + ds ds.Datastore +} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) SetUpTest(c *C) { + ks.dir = c.MkDir() + ks.ds, _ = fs.NewDatastore(ks.dir) +} + +func (ks *DSSuite) TestOpen(c *C) { + _, err := fs.NewDatastore("/tmp/foo/bar/baz") + c.Assert(err, Not(Equals), nil) + + // setup ds + _, err = fs.NewDatastore(ks.dir) + c.Assert(err, Equals, nil) +} + +func (ks *DSSuite) TestBasic(c *C) { + + keys := strsToKeys([]string{ + "foo", + "foo/bar", + "foo/bar/baz", + "foo/barb", + "foo/bar/bazb", + "foo/bar/baz/barb", + }) + + for _, k := range keys { + err := ks.ds.Put(k, []byte(k.String())) + c.Check(err, Equals, nil) + } + + for _, k := range keys { + v, err := ks.ds.Get(k) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true) + } +} + +func strsToKeys(strs []string) []ds.Key { + keys := make([]ds.Key, len(strs)) + for i, s := range strs { + keys[i] = ds.NewKey(s) + } + return keys +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go new file mode 100644 index 000000000..338b2e053 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go @@ -0,0 +1,44 @@ +package leveldb + +import ( + "bytes" + "io" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// CastAsReader does type assertions to find the type of a value and attempts +// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType +func CastAsReader(value interface{}) (io.Reader, error) { + switch v := value.(type) { + case io.Reader: + return v, nil + + case []byte: + return bytes.NewReader(v), nil + + case string: + return bytes.NewReader([]byte(v)), nil + + default: + return nil, ds.ErrInvalidType + } +} + +// // CastAsWriter does type assertions to find the type of a value and attempts +// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType +// func CastAsWriter(value interface{}) (err error) { +// switch v := value.(type) { +// case io.Reader: +// return v, nil +// +// case []byte: +// return bytes.NewReader(v), nil +// +// case string: +// return bytes.NewReader([]byte(v)), nil +// +// default: +// return nil, ds.ErrInvalidType +// } +// } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go index df210523f..0fc2ec054 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go @@ -2,12 +2,13 @@ package datastore_test import ( "bytes" - . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - . "launchpad.net/gocheck" "math/rand" "path" "strings" "testing" + + . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + . "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go index 1b00d8bfd..6fc2594a1 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go @@ -13,7 +13,7 @@ type Datastore struct { type Options opt.Options -func NewDatastore(path string, opts *Options) (*Datastore, error) { +func NewDatastore(path string, opts *Options) (ds.ThreadSafeDatastore, error) { var nopts opt.Options if opts != nil { nopts = opt.Options(*opts) @@ -76,3 +76,5 @@ func (d *Datastore) KeyList() ([]ds.Key, error) { func (d *Datastore) Close() (err error) { return d.DB.Close() } + +func (d *Datastore) IsThreadSafe() {} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go new file mode 100644 index 000000000..2dd74faa9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go @@ -0,0 +1,54 @@ +package lru + +import ( + "errors" + + lru "github.com/hashicorp/golang-lru" + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// Datastore uses golang-lru for internal storage. +type Datastore struct { + cache *lru.Cache +} + +// NewDatastore constructs a new LRU Datastore with given capacity. +func NewDatastore(capacity int) (*Datastore, error) { + cache, err := lru.New(capacity) + if err != nil { + return nil, err + } + + return &Datastore{cache: cache}, nil +} + +// Put stores the object `value` named by `key`. +func (d *Datastore) Put(key ds.Key, value interface{}) (err error) { + d.cache.Add(key, value) + return nil +} + +// Get retrieves the object `value` named by `key`. +func (d *Datastore) Get(key ds.Key) (value interface{}, err error) { + val, ok := d.cache.Get(key) + if !ok { + return nil, ds.ErrNotFound + } + return val, nil +} + +// Has returns whether the `key` is mapped to a `value`. +func (d *Datastore) Has(key ds.Key) (exists bool, err error) { + return ds.GetBackedHas(d, key) +} + +// Delete removes the value for given `key`. +func (d *Datastore) Delete(key ds.Key) (err error) { + d.cache.Remove(key) + return nil +} + +// KeyList returns a list of keys in the datastore +func (d *Datastore) KeyList() ([]ds.Key, error) { + return nil, errors.New("KeyList not implemented.") +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go new file mode 100644 index 000000000..b030df9a2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go @@ -0,0 +1,52 @@ +package lru_test + +import ( + "strconv" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + lru "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru" + . "gopkg.in/check.v1" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct{} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) TestBasic(c *C) { + var size = 1000 + + d, err := lru.NewDatastore(size) + c.Check(err, Equals, nil) + + for i := 0; i < size; i++ { + err := d.Put(ds.NewKey(strconv.Itoa(i)), i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i))) + c.Check(j, Equals, i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i))) + c.Check(j, Equals, nil) + c.Check(err, Equals, ds.ErrNotFound) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i + size))) + c.Check(j, Equals, i) + c.Check(err, Equals, nil) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go new file mode 100644 index 000000000..2f89def7c --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go @@ -0,0 +1,19 @@ +package datastore + +// type KeyIterator struct { +// HasNext() bool +// Next() interface{} +// } + +// type Query struct { +// } + +/* +QueryDatastores support a Query interface. Queries are used to support +searching for values (beyond simple key-based `Get`s). +*/ +// type QueryDatastore interface { +// // Query returns an Iterator of Keys whose Values match criteria +// // expressed in `query`. +// Query(Query) (iter Iterator, err error) +// } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go new file mode 100644 index 000000000..06a363410 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go @@ -0,0 +1,64 @@ +package sync + +import ( + "sync" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// MutexDatastore contains a child datastire and a mutex. +// used for coarse sync +type MutexDatastore struct { + sync.RWMutex + + child ds.Datastore +} + +// MutexWrap constructs a datastore with a coarse lock around +// the entire datastore, for every single operation +func MutexWrap(d ds.Datastore) ds.ThreadSafeDatastore { + return &MutexDatastore{child: d} +} + +// Children implements Shim +func (d *MutexDatastore) Children() []ds.Datastore { + return []ds.Datastore{d.child} +} + +// IsThreadSafe implements ThreadSafeDatastore +func (d *MutexDatastore) IsThreadSafe() {} + +// Put implements Datastore.Put +func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) { + d.Lock() + defer d.Unlock() + return d.child.Put(key, value) +} + +// Get implements Datastore.Get +func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) { + d.RLock() + defer d.RUnlock() + return d.child.Get(key) +} + +// Has implements Datastore.Has +func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) { + d.RLock() + defer d.RUnlock() + return d.child.Has(key) +} + +// Delete implements Datastore.Delete +func (d *MutexDatastore) Delete(key ds.Key) (err error) { + d.Lock() + defer d.Unlock() + return d.child.Delete(key) +} + +// KeyList implements Datastore.KeyList +func (d *MutexDatastore) KeyList() ([]ds.Key, error) { + d.RLock() + defer d.RUnlock() + return d.child.KeyList() +} From 88d804e32a90b7c3ed9e464bdaadab4761d65342 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:45:00 -0700 Subject: [PATCH 2/4] added core logging --- core/core.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/core.go b/core/core.go index dd5887ef5..decec6307 100644 --- a/core/core.go +++ b/core/core.go @@ -9,6 +9,7 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" bserv "github.com/jbenet/go-ipfs/blockservice" config "github.com/jbenet/go-ipfs/config" @@ -27,6 +28,8 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +var log = logging.MustGetLogger("core") + // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { From 2ce9415c6934ba882f16302620c2ac5f93c2a9a3 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:45:15 -0700 Subject: [PATCH 3/4] + fs ds + thread safe --- core/core.go | 2 +- core/datastore.go | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/core/core.go b/core/core.go index decec6307..8fa209bba 100644 --- a/core/core.go +++ b/core/core.go @@ -43,7 +43,7 @@ type IpfsNode struct { Peerstore peer.Peerstore // the local datastore - Datastore ds.Datastore + Datastore ds.ThreadSafeDatastore // the network message stream Network inet.Network diff --git a/core/datastore.go b/core/datastore.go index 9105adaab..30f92c8e8 100644 --- a/core/datastore.go +++ b/core/datastore.go @@ -4,11 +4,13 @@ import ( "fmt" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + fsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs" lds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb" + syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync" config "github.com/jbenet/go-ipfs/config" ) -func makeDatastore(cfg config.Datastore) (ds.Datastore, error) { +func makeDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { if len(cfg.Type) == 0 { return nil, fmt.Errorf("config datastore.type required") } @@ -16,14 +18,23 @@ func makeDatastore(cfg config.Datastore) (ds.Datastore, error) { switch cfg.Type { case "leveldb": return makeLevelDBDatastore(cfg) + case "memory": - return ds.NewMapDatastore(), nil + return syncds.MutexWrap(ds.NewMapDatastore()), nil + + case "fs": + log.Warning("using fs.Datastore at .datastore for testing.") + d, err := fsds.NewDatastore(".datastore") // for testing!! + if err != nil { + return nil, err + } + return syncds.MutexWrap(d), nil } return nil, fmt.Errorf("Unknown datastore type: %s", cfg.Type) } -func makeLevelDBDatastore(cfg config.Datastore) (ds.Datastore, error) { +func makeLevelDBDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { if len(cfg.Path) == 0 { return nil, fmt.Errorf("config datastore.path required for leveldb") } From 612be596438bde0a653e7965686a396b2050cadc Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 15:34:08 -0700 Subject: [PATCH 4/4] use string datastore keys. --- blockservice/blockservice.go | 8 +++----- blockstore/blockstore.go | 8 ++------ blockstore/blockstore_test.go | 2 +- peer/peerstore.go | 8 ++++---- routing/dht/dht.go | 16 +++++++++++----- routing/dht/handlers.go | 7 ++++--- routing/mock/routing.go | 4 ++-- util/util.go | 12 ++++++++++++ 8 files changed, 39 insertions(+), 26 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 36c3ed607..369cacc2a 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -7,7 +7,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" blocks "github.com/jbenet/go-ipfs/blocks" exchange "github.com/jbenet/go-ipfs/exchange" @@ -37,11 +37,10 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err // AddBlock adds a particular block to the service, Putting it into the datastore. func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() - dsk := ds.NewKey(string(k)) log.Debug("storing [%s] in datastore", k.Pretty()) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter - err := s.Datastore.Put(dsk, b.Data) + err := s.Datastore.Put(k.DsKey(), b.Data) if err != nil { return k, err } @@ -56,8 +55,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { log.Debug("BlockService GetBlock: '%s'", k.Pretty()) - dsk := ds.NewKey(string(k)) - datai, err := s.Datastore.Get(dsk) + datai, err := s.Datastore.Get(k.DsKey()) if err == nil { log.Debug("Blockservice: Got data in datastore.") bdata, ok := datai.([]byte) diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index a4fc1f65d..860d7ba41 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -27,7 +27,7 @@ type blockstore struct { } func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { - maybeData, err := bs.datastore.Get(toDatastoreKey(k)) + maybeData, err := bs.datastore.Get(k.DsKey()) if err != nil { return nil, err } @@ -39,9 +39,5 @@ func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { } func (bs *blockstore) Put(block blocks.Block) error { - return bs.datastore.Put(toDatastoreKey(block.Key()), block.Data) -} - -func toDatastoreKey(k u.Key) ds.Key { - return ds.NewKey(string(k)) + return bs.datastore.Put(block.Key().DsKey(), block.Data) } diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go index 4b0909d75..dfb9783af 100644 --- a/blockstore/blockstore_test.go +++ b/blockstore/blockstore_test.go @@ -44,7 +44,7 @@ func TestValueTypeMismatch(t *testing.T) { block := testutil.NewBlockOrFail(t, "some data") datastore := ds.NewMapDatastore() - datastore.Put(toDatastoreKey(block.Key()), "data that isn't a block!") + datastore.Put(block.Key().DsKey(), "data that isn't a block!") blockstore := NewBlockstore(datastore) diff --git a/peer/peerstore.go b/peer/peerstore.go index 9c0f28df3..e8eb0eac0 100644 --- a/peer/peerstore.go +++ b/peer/peerstore.go @@ -37,7 +37,7 @@ func (p *peerstore) Get(i ID) (*Peer, error) { p.RLock() defer p.RUnlock() - k := ds.NewKey(string(i)) + k := u.Key(i).DsKey() val, err := p.peers.Get(k) if err != nil { return nil, err @@ -54,7 +54,7 @@ func (p *peerstore) Put(peer *Peer) error { p.Lock() defer p.Unlock() - k := ds.NewKey(string(peer.ID)) + k := u.Key(peer.ID).DsKey() return p.peers.Put(k, peer) } @@ -62,7 +62,7 @@ func (p *peerstore) Delete(i ID) error { p.Lock() defer p.Unlock() - k := ds.NewKey(string(i)) + k := u.Key(i).DsKey() return p.peers.Delete(k) } @@ -84,7 +84,7 @@ func (p *peerstore) All() (*Map, error) { pval, ok := val.(*Peer) if ok { - (*ps)[u.Key(k.String())] = pval + (*ps)[u.Key(pval.ID)] = pval } } return ps, nil diff --git a/routing/dht/dht.go b/routing/dht/dht.go index f02be4711..aa7361e53 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -13,11 +13,11 @@ import ( peer "github.com/jbenet/go-ipfs/peer" kb "github.com/jbenet/go-ipfs/routing/kbucket" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) @@ -328,7 +328,7 @@ func (dht *IpfsDHT) getFromPeerList(ctx context.Context, key u.Key, func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { dht.dslock.Lock() defer dht.dslock.Unlock() - v, err := dht.datastore.Get(ds.NewKey(string(key))) + v, err := dht.datastore.Get(key.DsKey()) if err != nil { return nil, err } @@ -341,7 +341,7 @@ func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { } func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { - return dht.datastore.Put(ds.NewKey(string(key)), value) + return dht.datastore.Put(key.DsKey(), value) } // Update signals to all routingTables to Update their last-seen status @@ -494,13 +494,19 @@ func (dht *IpfsDHT) ensureConnectedToPeer(pbp *Message_Peer) (*peer.Peer, error) return p, err } +//TODO: this should be smarter about which keys it selects. func (dht *IpfsDHT) loadProvidableKeys() error { kl, err := dht.datastore.KeyList() if err != nil { return err } - for _, k := range kl { - dht.providers.AddProvider(u.Key(k.Bytes()), dht.self) + for _, dsk := range kl { + k := u.KeyFromDsKey(dsk) + if len(k) == 0 { + log.Error("loadProvidableKeys error: %v", dsk) + } + + dht.providers.AddProvider(k, dht.self) } return nil } diff --git a/routing/dht/handlers.go b/routing/dht/handlers.go index 4301d1e4e..ac03ed3e8 100644 --- a/routing/dht/handlers.go +++ b/routing/dht/handlers.go @@ -51,7 +51,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error // let's first check if we have the value locally. u.DOut("[%s] handleGetValue looking into ds\n", dht.self.ID.Pretty()) - dskey := ds.NewKey(pmes.GetKey()) + dskey := u.Key(pmes.GetKey()).DsKey() iVal, err := dht.datastore.Get(dskey) u.DOut("[%s] handleGetValue looking into ds GOT %v\n", dht.self.ID.Pretty(), iVal) @@ -96,7 +96,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error func (dht *IpfsDHT) handlePutValue(p *peer.Peer, pmes *Message) (*Message, error) { dht.dslock.Lock() defer dht.dslock.Unlock() - dskey := ds.NewKey(pmes.GetKey()) + dskey := u.Key(pmes.GetKey()).DsKey() err := dht.datastore.Put(dskey, pmes.GetValue()) u.DOut("[%s] handlePutValue %v %v\n", dht.self.ID.Pretty(), dskey, pmes.GetValue()) return pmes, err @@ -137,7 +137,8 @@ func (dht *IpfsDHT) handleGetProviders(p *peer.Peer, pmes *Message) (*Message, e resp := newMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel()) // check if we have this value, to add ourselves as provider. - has, err := dht.datastore.Has(ds.NewKey(pmes.GetKey())) + dsk := u.Key(pmes.GetKey()).DsKey() + has, err := dht.datastore.Has(dsk) if err != nil && err != ds.ErrNotFound { u.PErr("unexpected datastore error: %v\n", err) has = false diff --git a/routing/mock/routing.go b/routing/mock/routing.go index e5fdb96fc..954914c3b 100644 --- a/routing/mock/routing.go +++ b/routing/mock/routing.go @@ -33,11 +33,11 @@ func (mr *MockRouter) SetRoutingServer(rs RoutingServer) { } func (mr *MockRouter) PutValue(ctx context.Context, key u.Key, val []byte) error { - return mr.datastore.Put(ds.NewKey(string(key)), val) + return mr.datastore.Put(key.DsKey(), val) } func (mr *MockRouter) GetValue(ctx context.Context, key u.Key) ([]byte, error) { - v, err := mr.datastore.Get(ds.NewKey(string(key))) + v, err := mr.datastore.Get(key.DsKey()) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index 40d04ed73..8831c6831 100644 --- a/util/util.go +++ b/util/util.go @@ -41,6 +41,18 @@ func (k Key) Pretty() string { return b58.Encode([]byte(k)) } +// DsKey returns a Datastore key +func (k Key) DsKey() ds.Key { + return ds.NewKey(k.Pretty()) +} + +// KeyFromDsKey returns a Datastore key +func KeyFromDsKey(dsk ds.Key) Key { + l := dsk.List() + enc := l[len(l)-1] + return Key(b58.Decode(enc)) +} + // Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits func Hash(data []byte) (mh.Multihash, error) { return mh.Sum(data, mh.SHA2_256, -1)