Merge branch 'ipns-dev' of github.com:jbenet/go-ipfs into ipns-dev

Godeps/Godeps.json

@@ -64,7 +64,7 @@
 		},
 		{
 			"ImportPath": "github.com/jbenet/datastore.go",
-			"Rev": "e89f0511689bb2d0608496e15491f241842de085"
+			"Rev": "e7d6f7cb9e3c207a04c5397c449d10a6f9d403a0"
 		},
 		{
 			"ImportPath": "github.com/jbenet/go-base58",

Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json

@@ -0,0 +1,43 @@
+{
+	"ImportPath": "github.com/jbenet/datastore.go",
+	"GoVersion": "go1.3.1",
+	"Packages": [
+		"./..."
+	],
+	"Deps": [
+		{
+			"ImportPath": "code.google.com/p/go-uuid/uuid",
+			"Comment": "null-12",
+			"Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9"
+		},
+		{
+			"ImportPath": "code.google.com/p/snappy-go/snappy",
+			"Comment": "null-15",
+			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
+		},
+		{
+			"ImportPath": "github.com/codahale/blake2",
+			"Rev": "3fa823583afba430e8fc7cdbcc670dbf90bfacc4"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/golang-lru",
+			"Rev": "4dfff096c4973178c8f35cf6dd1a732a0a139370"
+		},
+		{
+			"ImportPath": "github.com/mattbaird/elastigo/api",
+			"Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
+		},
+		{
+			"ImportPath": "github.com/mattbaird/elastigo/core",
+			"Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
+		},
+		{
+			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
+			"Rev": "9bca75c48d6c31becfbb127702b425e7226052e3"
+		},
+		{
+			"ImportPath": "gopkg.in/check.v1",
+			"Rev": "91ae5f88a67b14891cfd43895b01164f6c120420"
+		}
+	]
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme

@@ -0,0 +1,5 @@
+This directory tree is generated automatically by godep.
+
+Please do not edit.
+
+See https://github.com/tools/godep for more information.

Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go

@@ -1,28 +1,30 @@
 package datastore
 
-import (
-	"log"
-)
+import "log"
 
 // Here are some basic datastore implementations.
 
-// MapDatastore uses a standard Go map for internal storage.
 type keyMap map[Key]interface{}
+
+// MapDatastore uses a standard Go map for internal storage.
 type MapDatastore struct {
 	values keyMap
 }
 
+// NewMapDatastore constructs a MapDatastore
 func NewMapDatastore() (d *MapDatastore) {
 	return &MapDatastore{
 		values: keyMap{},
 	}
 }
 
+// Put implements Datastore.Put
 func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
 	d.values[key] = value
 	return nil
 }
 
+// Get implements Datastore.Get
 func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
 	val, found := d.values[key]
 	if !found {
@@ -31,19 +33,22 @@ func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
 	return val, nil
 }
 
+// Has implements Datastore.Has
 func (d *MapDatastore) Has(key Key) (exists bool, err error) {
 	_, found := d.values[key]
 	return found, nil
 }
 
+// Delete implements Datastore.Delete
 func (d *MapDatastore) Delete(key Key) (err error) {
 	delete(d.values, key)
 	return nil
 }
 
+// KeyList implements Datastore.KeyList
 func (d *MapDatastore) KeyList() ([]Key, error) {
 	var keys []Key
-	for k, _ := range d.values {
+	for k := range d.values {
 		keys = append(keys, k)
 	}
 	return keys, nil
@@ -54,26 +59,32 @@ func (d *MapDatastore) KeyList() ([]Key, error) {
 type NullDatastore struct {
 }
 
+// NewNullDatastore constructs a null datastoe
 func NewNullDatastore() *NullDatastore {
 	return &NullDatastore{}
 }
 
+// Put implements Datastore.Put
 func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
 	return nil
 }
 
+// Get implements Datastore.Get
 func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
 	return nil, nil
 }
 
+// Has implements Datastore.Has
 func (d *NullDatastore) Has(key Key) (exists bool, err error) {
 	return false, nil
 }
 
+// Delete implements Datastore.Delete
 func (d *NullDatastore) Delete(key Key) (err error) {
 	return nil
 }
 
+// KeyList implements Datastore.KeyList
 func (d *NullDatastore) KeyList() ([]Key, error) {
 	return nil, nil
 }
@@ -81,38 +92,56 @@ func (d *NullDatastore) KeyList() ([]Key, error) {
 // LogDatastore logs all accesses through the datastore.
 type LogDatastore struct {
 	Name  string
-	Child Datastore
+	child Datastore
 }
 
-func NewLogDatastore(ds Datastore, name string) *LogDatastore {
+// Shim is a datastore which has a child.
+type Shim interface {
+	Datastore
+
+	Children() []Datastore
+}
+
+// NewLogDatastore constructs a log datastore.
+func NewLogDatastore(ds Datastore, name string) Shim {
 	if len(name) < 1 {
 		name = "LogDatastore"
 	}
-	return &LogDatastore{Name: name, Child: ds}
+	return &LogDatastore{Name: name, child: ds}
 }
 
+// Children implements Shim
+func (d *LogDatastore) Children() []Datastore {
+	return []Datastore{d.child}
+}
+
+// Put implements Datastore.Put
 func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
-	log.Printf("%s: Put %s", d.Name, key)
+	log.Printf("%s: Put %s\n", d.Name, key)
 	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
-	return d.Child.Put(key, value)
+	return d.child.Put(key, value)
 }
 
+// Get implements Datastore.Get
 func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
-	log.Printf("%s: Get %s", d.Name, key)
-	return d.Child.Get(key)
+	log.Printf("%s: Get %s\n", d.Name, key)
+	return d.child.Get(key)
 }
 
+// Has implements Datastore.Has
 func (d *LogDatastore) Has(key Key) (exists bool, err error) {
-	log.Printf("%s: Has %s", d.Name, key)
-	return d.Child.Has(key)
+	log.Printf("%s: Has %s\n", d.Name, key)
+	return d.child.Has(key)
 }
 
+// Delete implements Datastore.Delete
 func (d *LogDatastore) Delete(key Key) (err error) {
-	log.Printf("%s: Delete %s", d.Name, key)
-	return d.Child.Delete(key)
+	log.Printf("%s: Delete %s\n", d.Name, key)
+	return d.child.Delete(key)
 }
 
+// KeyList implements Datastore.KeyList
 func (d *LogDatastore) KeyList() ([]Key, error) {
-	log.Printf("%s: Get KeyList.", d.Name)
-	return d.Child.KeyList()
+	log.Printf("%s: Get KeyList\n", d.Name)
+	return d.child.KeyList()
 }

Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go

@@ -0,0 +1,13 @@
+package datastore_test
+
+import (
+	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type BasicSuite struct{}
+
+var _ = Suite(&BasicSuite{})

Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go

@@ -5,7 +5,7 @@ import (
 )
 
 /*
-A Datastore represents storage for any key-value pair.
+Datastore represents storage for any key-value pair.
 
 Datastores are general enough to be backed by all kinds of different storage:
 in-memory caches, databases, a remote datastore, flat files on disk, etc.
@@ -27,7 +27,6 @@ and thus it should behave predictably and handle exceptional conditions with
 proper error reporting. Thus, all Datastore calls may return errors, which
 should be checked by callers.
 */
-
 type Datastore interface {
 	// Put stores the object `value` named by `key`.
 	//
@@ -53,20 +52,27 @@ type Datastore interface {
 	// Delete removes the value for given `key`.
 	Delete(key Key) (err error)
 
-	// Returns a list of keys in the datastore
+	// KeyList returns a list of keys in the datastore
 	KeyList() ([]Key, error)
 }
 
+// ThreadSafeDatastore is an interface that all threadsafe datastore should
+// implement to leverage type safety checks.
+type ThreadSafeDatastore interface {
+	Datastore
+	IsThreadSafe()
+}
+
 // Errors
 
 // ErrNotFound is returned by Get, Has, and Delete when a datastore does not
 // map the given key to a value.
-var ErrNotFound = errors.New("datastore: key not found.")
+var ErrNotFound = errors.New("datastore: key not found")
 
 // ErrInvalidType is returned by Put when a given value is incopatible with
 // the type the datastore supports. This means a conversion (or serialization)
 // is needed beforehand.
-var ErrInvalidType = errors.New("datastore: invalid type error.")
+var ErrInvalidType = errors.New("datastore: invalid type error")
 
 // GetBackedHas provides a default Datastore.Has implementation.
 // It exists so Datastore.Has implementations can use it, like so:

Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go

@@ -0,0 +1,122 @@
+package fs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// Datastore uses a standard Go map for internal storage.
+type Datastore struct {
+	path string
+}
+
+// NewDatastore returns a new fs Datastore at given `path`
+func NewDatastore(path string) (ds.Datastore, error) {
+	if !isDir(path) {
+		return nil, fmt.Errorf("Failed to find directory at: %v (file? perms?)", path)
+	}
+
+	return &Datastore{path: path}, nil
+}
+
+// KeyFilename returns the filename associated with `key`
+func (d *Datastore) KeyFilename(key ds.Key) string {
+	return filepath.Join(d.path, key.String(), ".dsobject")
+}
+
+// Put stores the given value.
+func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
+
+	// TODO: maybe use io.Readers/Writers?
+	// r, err := dsio.CastAsReader(value)
+	// if err != nil {
+	// 	return err
+	// }
+
+	val, ok := value.([]byte)
+	if !ok {
+		return ds.ErrInvalidType
+	}
+
+	fn := d.KeyFilename(key)
+
+	// mkdirall above.
+	err = os.MkdirAll(filepath.Dir(fn), 0755)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(fn, val, 0666)
+}
+
+// Get returns the value for given key
+func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
+	fn := d.KeyFilename(key)
+	if !isFile(fn) {
+		return nil, ds.ErrNotFound
+	}
+
+	return ioutil.ReadFile(fn)
+}
+
+// Has returns whether the datastore has a value for a given key
+func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
+	return ds.GetBackedHas(d, key)
+}
+
+// Delete removes the value for given key
+func (d *Datastore) Delete(key ds.Key) (err error) {
+	fn := d.KeyFilename(key)
+	if !isFile(fn) {
+		return ds.ErrNotFound
+	}
+
+	return os.Remove(fn)
+}
+
+// KeyList returns a list of all keys in the datastore
+func (d *Datastore) KeyList() ([]ds.Key, error) {
+
+	keys := []ds.Key{}
+
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		// remove ds path prefix
+		if strings.HasPrefix(path, d.path) {
+			path = path[len(d.path):]
+		}
+
+		if !info.IsDir() {
+			key := ds.NewKey(path)
+			keys = append(keys, key)
+		}
+		return nil
+	}
+
+	filepath.Walk(d.path, walkFn)
+	return keys, nil
+}
+
+// isDir returns whether given path is a directory
+func isDir(path string) bool {
+	finfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return finfo.IsDir()
+}
+
+// isFile returns whether given path is a file
+func isFile(path string) bool {
+	finfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return !finfo.IsDir()
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go

@@ -0,0 +1,65 @@
+package fs_test
+
+import (
+	"bytes"
+	"testing"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	fs "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs"
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type DSSuite struct {
+	dir string
+	ds  ds.Datastore
+}
+
+var _ = Suite(&DSSuite{})
+
+func (ks *DSSuite) SetUpTest(c *C) {
+	ks.dir = c.MkDir()
+	ks.ds, _ = fs.NewDatastore(ks.dir)
+}
+
+func (ks *DSSuite) TestOpen(c *C) {
+	_, err := fs.NewDatastore("/tmp/foo/bar/baz")
+	c.Assert(err, Not(Equals), nil)
+
+	// setup ds
+	_, err = fs.NewDatastore(ks.dir)
+	c.Assert(err, Equals, nil)
+}
+
+func (ks *DSSuite) TestBasic(c *C) {
+
+	keys := strsToKeys([]string{
+		"foo",
+		"foo/bar",
+		"foo/bar/baz",
+		"foo/barb",
+		"foo/bar/bazb",
+		"foo/bar/baz/barb",
+	})
+
+	for _, k := range keys {
+		err := ks.ds.Put(k, []byte(k.String()))
+		c.Check(err, Equals, nil)
+	}
+
+	for _, k := range keys {
+		v, err := ks.ds.Get(k)
+		c.Check(err, Equals, nil)
+		c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true)
+	}
+}
+
+func strsToKeys(strs []string) []ds.Key {
+	keys := make([]ds.Key, len(strs))
+	for i, s := range strs {
+		keys[i] = ds.NewKey(s)
+	}
+	return keys
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go

@@ -0,0 +1,44 @@
+package leveldb
+
+import (
+	"bytes"
+	"io"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// CastAsReader does type assertions to find the type of a value and attempts
+// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType
+func CastAsReader(value interface{}) (io.Reader, error) {
+	switch v := value.(type) {
+	case io.Reader:
+		return v, nil
+
+	case []byte:
+		return bytes.NewReader(v), nil
+
+	case string:
+		return bytes.NewReader([]byte(v)), nil
+
+	default:
+		return nil, ds.ErrInvalidType
+	}
+}
+
+// // CastAsWriter does type assertions to find the type of a value and attempts
+// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType
+// func CastAsWriter(value interface{}) (err error) {
+// 	switch v := value.(type) {
+// 	case io.Reader:
+// 		return v, nil
+//
+// 	case []byte:
+// 		return bytes.NewReader(v), nil
+//
+// 	case string:
+// 		return bytes.NewReader([]byte(v)), nil
+//
+// 	default:
+// 		return nil, ds.ErrInvalidType
+// 	}
+// }

Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go

@@ -2,12 +2,13 @@ package datastore_test
 
 import (
 	"bytes"
-	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
-	. "launchpad.net/gocheck"
 	"math/rand"
 	"path"
 	"strings"
 	"testing"
+
+	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	. "gopkg.in/check.v1"
 )
 
 // Hook up gocheck into the "go test" runner.

Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go

@@ -13,7 +13,7 @@ type Datastore struct {
 
 type Options opt.Options
 
-func NewDatastore(path string, opts *Options) (*Datastore, error) {
+func NewDatastore(path string, opts *Options) (ds.ThreadSafeDatastore, error) {
 	var nopts opt.Options
 	if opts != nil {
 		nopts = opt.Options(*opts)
@@ -76,3 +76,5 @@ func (d *Datastore) KeyList() ([]ds.Key, error) {
 func (d *Datastore) Close() (err error) {
 	return d.DB.Close()
 }
+
+func (d *Datastore) IsThreadSafe() {}

Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go

@@ -0,0 +1,54 @@
+package lru
+
+import (
+	"errors"
+
+	lru "github.com/hashicorp/golang-lru"
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// Datastore uses golang-lru for internal storage.
+type Datastore struct {
+	cache *lru.Cache
+}
+
+// NewDatastore constructs a new LRU Datastore with given capacity.
+func NewDatastore(capacity int) (*Datastore, error) {
+	cache, err := lru.New(capacity)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Datastore{cache: cache}, nil
+}
+
+// Put stores the object `value` named by `key`.
+func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
+	d.cache.Add(key, value)
+	return nil
+}
+
+// Get retrieves the object `value` named by `key`.
+func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
+	val, ok := d.cache.Get(key)
+	if !ok {
+		return nil, ds.ErrNotFound
+	}
+	return val, nil
+}
+
+// Has returns whether the `key` is mapped to a `value`.
+func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
+	return ds.GetBackedHas(d, key)
+}
+
+// Delete removes the value for given `key`.
+func (d *Datastore) Delete(key ds.Key) (err error) {
+	d.cache.Remove(key)
+	return nil
+}
+
+// KeyList returns a list of keys in the datastore
+func (d *Datastore) KeyList() ([]ds.Key, error) {
+	return nil, errors.New("KeyList not implemented.")
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go

@@ -0,0 +1,52 @@
+package lru_test
+
+import (
+	"strconv"
+	"testing"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	lru "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru"
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type DSSuite struct{}
+
+var _ = Suite(&DSSuite{})
+
+func (ks *DSSuite) TestBasic(c *C) {
+	var size = 1000
+
+	d, err := lru.NewDatastore(size)
+	c.Check(err, Equals, nil)
+
+	for i := 0; i < size; i++ {
+		err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
+		c.Check(j, Equals, i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
+		c.Check(j, Equals, nil)
+		c.Check(err, Equals, ds.ErrNotFound)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
+		c.Check(j, Equals, i)
+		c.Check(err, Equals, nil)
+	}
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go

@@ -0,0 +1,19 @@
+package datastore
+
+// type KeyIterator struct {
+//   HasNext() bool
+//   Next() interface{}
+// }
+
+// type Query struct {
+// }
+
+/*
+QueryDatastores support a Query interface. Queries are used to support
+searching for values (beyond simple key-based `Get`s).
+*/
+// type QueryDatastore interface {
+//   // Query returns an Iterator of Keys whose Values match criteria
+//   // expressed in `query`.
+//   Query(Query) (iter Iterator, err error)
+// }

Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go

@@ -0,0 +1,64 @@
+package sync
+
+import (
+	"sync"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// MutexDatastore contains a child datastire and a mutex.
+// used for coarse sync
+type MutexDatastore struct {
+	sync.RWMutex
+
+	child ds.Datastore
+}
+
+// MutexWrap constructs a datastore with a coarse lock around
+// the entire datastore, for every single operation
+func MutexWrap(d ds.Datastore) ds.ThreadSafeDatastore {
+	return &MutexDatastore{child: d}
+}
+
+// Children implements Shim
+func (d *MutexDatastore) Children() []ds.Datastore {
+	return []ds.Datastore{d.child}
+}
+
+// IsThreadSafe implements ThreadSafeDatastore
+func (d *MutexDatastore) IsThreadSafe() {}
+
+// Put implements Datastore.Put
+func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) {
+	d.Lock()
+	defer d.Unlock()
+	return d.child.Put(key, value)
+}
+
+// Get implements Datastore.Get
+func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.Get(key)
+}
+
+// Has implements Datastore.Has
+func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.Has(key)
+}
+
+// Delete implements Datastore.Delete
+func (d *MutexDatastore) Delete(key ds.Key) (err error) {
+	d.Lock()
+	defer d.Unlock()
+	return d.child.Delete(key)
+}
+
+// KeyList implements Datastore.KeyList
+func (d *MutexDatastore) KeyList() ([]ds.Key, error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.KeyList()
+}

blockservice/blockservice.go

@@ -7,7 +7,7 @@ import (
 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
 	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
-	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
+	logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
 
 	blocks "github.com/jbenet/go-ipfs/blocks"
 	exchange "github.com/jbenet/go-ipfs/exchange"
@@ -37,11 +37,10 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err
 // AddBlock adds a particular block to the service, Putting it into the datastore.
 func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) {
 	k := b.Key()
-	dsk := ds.NewKey(string(k))
 	log.Debug("storing [%s] in datastore", k.Pretty())
 	// TODO(brian): define a block datastore with a Put method which accepts a
 	// block parameter
-	err := s.Datastore.Put(dsk, b.Data)
+	err := s.Datastore.Put(k.DsKey(), b.Data)
 	if err != nil {
 		return k, err
 	}
@@ -56,8 +55,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) {
 // Getting it from the datastore using the key (hash).
 func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) {
 	log.Debug("BlockService GetBlock: '%s'", k.Pretty())
-	dsk := ds.NewKey(string(k))
-	datai, err := s.Datastore.Get(dsk)
+	datai, err := s.Datastore.Get(k.DsKey())
 	if err == nil {
 		log.Debug("Blockservice: Got data in datastore.")
 		bdata, ok := datai.([]byte)

blockstore/blockstore.go

@@ -27,7 +27,7 @@ type blockstore struct {
 }
 
 func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) {
-	maybeData, err := bs.datastore.Get(toDatastoreKey(k))
+	maybeData, err := bs.datastore.Get(k.DsKey())
 	if err != nil {
 		return nil, err
 	}
@@ -39,9 +39,5 @@ func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) {
 }
 
 func (bs *blockstore) Put(block blocks.Block) error {
-	return bs.datastore.Put(toDatastoreKey(block.Key()), block.Data)
-}
-
-func toDatastoreKey(k u.Key) ds.Key {
-	return ds.NewKey(string(k))
+	return bs.datastore.Put(block.Key().DsKey(), block.Data)
 }

blockstore/blockstore_test.go

@@ -44,7 +44,7 @@ func TestValueTypeMismatch(t *testing.T) {
 	block := testutil.NewBlockOrFail(t, "some data")
 
 	datastore := ds.NewMapDatastore()
-	datastore.Put(toDatastoreKey(block.Key()), "data that isn't a block!")
+	datastore.Put(block.Key().DsKey(), "data that isn't a block!")
 
 	blockstore := NewBlockstore(datastore)
 

core/core.go

@@ -9,6 +9,7 @@ import (
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
 	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
 	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
+	logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
 
 	bserv "github.com/jbenet/go-ipfs/blockservice"
 	config "github.com/jbenet/go-ipfs/config"
@@ -27,6 +28,8 @@ import (
 	u "github.com/jbenet/go-ipfs/util"
 )
 
+var log = logging.MustGetLogger("core")
+
 // IpfsNode is IPFS Core module. It represents an IPFS instance.
 type IpfsNode struct {
 
@@ -40,7 +43,7 @@ type IpfsNode struct {
 	Peerstore peer.Peerstore
 
 	// the local datastore
-	Datastore ds.Datastore
+	Datastore ds.ThreadSafeDatastore
 
 	// the network message stream
 	Network inet.Network

core/datastore.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	fsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs"
 	lds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb"
+	syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync"
 	config "github.com/jbenet/go-ipfs/config"
 )
 
-func makeDatastore(cfg config.Datastore) (ds.Datastore, error) {
+func makeDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) {
 	if len(cfg.Type) == 0 {
 		return nil, fmt.Errorf("config datastore.type required")
 	}
@@ -16,14 +18,23 @@ func makeDatastore(cfg config.Datastore) (ds.Datastore, error) {
 	switch cfg.Type {
 	case "leveldb":
 		return makeLevelDBDatastore(cfg)
+
 	case "memory":
-		return ds.NewMapDatastore(), nil
+		return syncds.MutexWrap(ds.NewMapDatastore()), nil
+
+	case "fs":
+		log.Warning("using fs.Datastore at .datastore for testing.")
+		d, err := fsds.NewDatastore(".datastore") // for testing!!
+		if err != nil {
+			return nil, err
+		}
+		return syncds.MutexWrap(d), nil
 	}
 
 	return nil, fmt.Errorf("Unknown datastore type: %s", cfg.Type)
 }
 
-func makeLevelDBDatastore(cfg config.Datastore) (ds.Datastore, error) {
+func makeLevelDBDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) {
 	if len(cfg.Path) == 0 {
 		return nil, fmt.Errorf("config datastore.path required for leveldb")
 	}

peer/peerstore.go

@@ -37,7 +37,7 @@ func (p *peerstore) Get(i ID) (*Peer, error) {
 	p.RLock()
 	defer p.RUnlock()
 
-	k := ds.NewKey(string(i))
+	k := u.Key(i).DsKey()
 	val, err := p.peers.Get(k)
 	if err != nil {
 		return nil, err
@@ -54,7 +54,7 @@ func (p *peerstore) Put(peer *Peer) error {
 	p.Lock()
 	defer p.Unlock()
 
-	k := ds.NewKey(string(peer.ID))
+	k := u.Key(peer.ID).DsKey()
 	return p.peers.Put(k, peer)
 }
 
@@ -62,7 +62,7 @@ func (p *peerstore) Delete(i ID) error {
 	p.Lock()
 	defer p.Unlock()
 
-	k := ds.NewKey(string(i))
+	k := u.Key(i).DsKey()
 	return p.peers.Delete(k)
 }
 
@@ -84,7 +84,7 @@ func (p *peerstore) All() (*Map, error) {
 
 		pval, ok := val.(*Peer)
 		if ok {
-			(*ps)[u.Key(k.String())] = pval
+			(*ps)[u.Key(pval.ID)] = pval
 		}
 	}
 	return ps, nil

routing/dht/dht.go

@@ -13,11 +13,11 @@ import (
 	peer "github.com/jbenet/go-ipfs/peer"
 	kb "github.com/jbenet/go-ipfs/routing/kbucket"
 	u "github.com/jbenet/go-ipfs/util"
-	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
 
 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
 	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
+	logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
 
 	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
 )
@@ -328,7 +328,7 @@ func (dht *IpfsDHT) getFromPeerList(ctx context.Context, key u.Key,
 func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) {
 	dht.dslock.Lock()
 	defer dht.dslock.Unlock()
-	v, err := dht.datastore.Get(ds.NewKey(string(key)))
+	v, err := dht.datastore.Get(key.DsKey())
 	if err != nil {
 		return nil, err
 	}
@@ -341,7 +341,7 @@ func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) {
 }
 
 func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error {
-	return dht.datastore.Put(ds.NewKey(string(key)), value)
+	return dht.datastore.Put(key.DsKey(), value)
 }
 
 // Update signals to all routingTables to Update their last-seen status
@@ -494,13 +494,19 @@ func (dht *IpfsDHT) ensureConnectedToPeer(pbp *Message_Peer) (*peer.Peer, error)
 	return p, err
 }
 
+//TODO: this should be smarter about which keys it selects.
 func (dht *IpfsDHT) loadProvidableKeys() error {
 	kl, err := dht.datastore.KeyList()
 	if err != nil {
 		return err
 	}
-	for _, k := range kl {
-		dht.providers.AddProvider(u.Key(k.Bytes()), dht.self)
+	for _, dsk := range kl {
+		k := u.KeyFromDsKey(dsk)
+		if len(k) == 0 {
+			log.Error("loadProvidableKeys error: %v", dsk)
+		}
+
+		dht.providers.AddProvider(k, dht.self)
 	}
 	return nil
 }

routing/dht/handlers.go

@@ -51,7 +51,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error
 
 	// let's first check if we have the value locally.
 	u.DOut("[%s] handleGetValue looking into ds\n", dht.self.ID.Pretty())
-	dskey := ds.NewKey(pmes.GetKey())
+	dskey := u.Key(pmes.GetKey()).DsKey()
 	iVal, err := dht.datastore.Get(dskey)
 	u.DOut("[%s] handleGetValue looking into ds GOT %v\n", dht.self.ID.Pretty(), iVal)
 
@@ -96,7 +96,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error
 func (dht *IpfsDHT) handlePutValue(p *peer.Peer, pmes *Message) (*Message, error) {
 	dht.dslock.Lock()
 	defer dht.dslock.Unlock()
-	dskey := ds.NewKey(pmes.GetKey())
+	dskey := u.Key(pmes.GetKey()).DsKey()
 	err := dht.datastore.Put(dskey, pmes.GetValue())
 	u.DOut("[%s] handlePutValue %v %v\n", dht.self.ID.Pretty(), dskey, pmes.GetValue())
 	return pmes, err
@@ -137,7 +137,8 @@ func (dht *IpfsDHT) handleGetProviders(p *peer.Peer, pmes *Message) (*Message, e
 	resp := newMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())
 
 	// check if we have this value, to add ourselves as provider.
-	has, err := dht.datastore.Has(ds.NewKey(pmes.GetKey()))
+	dsk := u.Key(pmes.GetKey()).DsKey()
+	has, err := dht.datastore.Has(dsk)
 	if err != nil && err != ds.ErrNotFound {
 		u.PErr("unexpected datastore error: %v\n", err)
 		has = false

routing/mock/routing.go

@@ -33,11 +33,11 @@ func (mr *MockRouter) SetRoutingServer(rs RoutingServer) {
 }
 
 func (mr *MockRouter) PutValue(ctx context.Context, key u.Key, val []byte) error {
-	return mr.datastore.Put(ds.NewKey(string(key)), val)
+	return mr.datastore.Put(key.DsKey(), val)
 }
 
 func (mr *MockRouter) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
-	v, err := mr.datastore.Get(ds.NewKey(string(key)))
+	v, err := mr.datastore.Get(key.DsKey())
 	if err != nil {
 		return nil, err
 	}

util/util.go

@@ -41,6 +41,18 @@ func (k Key) Pretty() string {
 	return b58.Encode([]byte(k))
 }
 
+// DsKey returns a Datastore key
+func (k Key) DsKey() ds.Key {
+	return ds.NewKey(k.Pretty())
+}
+
+// KeyFromDsKey returns a Datastore key
+func KeyFromDsKey(dsk ds.Key) Key {
+	l := dsk.List()
+	enc := l[len(l)-1]
+	return Key(b58.Decode(enc))
+}
+
 // Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits
 func Hash(data []byte) (mh.Multihash, error) {
 	return mh.Sum(data, mh.SHA2_256, -1)