From 7875674caeb84acb52e4bc47598e40adbe2f401e Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 15 Mar 2016 13:46:03 -0700 Subject: [PATCH] make datastore configuration nicely customizable License: MIT Signed-off-by: Jeromy make things super customizable License: MIT Signed-off-by: Jeromy better json format License: MIT Signed-off-by: Jeromy Migrate to new flatfs License: MIT Signed-off-by: Jakub Sztandera --- repo/config/datastore.go | 35 ++++++----- repo/config/init.go | 39 +++++++++--- repo/fsrepo/datastores.go | 124 ++++++++++++++++++++++++++++++++++++++ repo/fsrepo/fsrepo.go | 13 ++-- 4 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 repo/fsrepo/datastores.go diff --git a/repo/config/datastore.go b/repo/config/datastore.go index 2b861a113..aec78e0e1 100644 --- a/repo/config/datastore.go +++ b/repo/config/datastore.go @@ -1,40 +1,43 @@ package config -import ( - "encoding/json" -) - // DefaultDataStoreDirectory is the directory to store all the local IPFS data. const DefaultDataStoreDirectory = "datastore" // Datastore tracks the configuration of the datastore. type Datastore struct { - Type string - Path string StorageMax string // in B, kB, kiB, MB, ... StorageGCWatermark int64 // in percentage to multiply on StorageMax GCPeriod string // in ns, us, ms, s, m, h + Path string + NoSync bool // deprecated + + Spec map[string]interface{} - Params *json.RawMessage - NoSync bool HashOnRead bool BloomFilterSize int } -func (d *Datastore) ParamData() []byte { - if d.Params == nil { - return nil - } - - return []byte(*d.Params) -} - type S3Datastore struct { Region string `json:"region"` Bucket string `json:"bucket"` ACL string `json:"acl"` } +type FlatDS struct { + Path string + ShardFunc string + Sync bool +} + +type LevelDB struct { + Path string + Compression string +} + +type SbsDS struct { + Path string +} + // DataStorePath returns the default data store path given a configuration root // (set an empty string to have the default configuration root) func DataStorePath(configroot string) (string, error) { diff --git a/repo/config/init.go b/repo/config/init.go index f31edd42b..ae41b68e9 100644 --- a/repo/config/init.go +++ b/repo/config/init.go @@ -42,7 +42,7 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { Gateway: "/ip4/127.0.0.1/tcp/8080", }, - Datastore: datastore, + Datastore: *datastore, Bootstrap: BootstrapPeerStrings(bootstrapPeers), Identity: identity, Discovery: Discovery{MDNS{ @@ -79,19 +79,38 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { return conf, nil } -func datastoreConfig() (Datastore, error) { - dspath, err := DataStorePath("") - if err != nil { - return Datastore{}, err - } - return Datastore{ - Path: dspath, - Type: "leveldb", +func datastoreConfig() (*Datastore, error) { + return &Datastore{ StorageMax: "10GB", StorageGCWatermark: 90, // 90% GCPeriod: "1h", - HashOnRead: false, BloomFilterSize: 0, + Spec: map[string]interface{}{ + "type": "mount", + "mounts": []interface{}{ + map[string]interface{}{ + "mountpoint": "/blocks", + "type": "measure", + "prefix": "flatfs.datastore", + "child": map[string]interface{}{ + "type": "flatfs", + "path": "blocks", + "nosync": false, + "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", + }, + }, + map[string]interface{}{ + "mountpoint": "/", + "type": "measure", + "prefix": "leveldb.datastore", + "child": map[string]interface{}{ + "type": "levelds", + "path": "datastore", + "compression": "none", + }, + }, + }, + }, }, nil } diff --git a/repo/fsrepo/datastores.go b/repo/fsrepo/datastores.go new file mode 100644 index 000000000..95fa04503 --- /dev/null +++ b/repo/fsrepo/datastores.go @@ -0,0 +1,124 @@ +package fsrepo + +import ( + "encoding/json" + "fmt" + "path/filepath" + + repo "github.com/ipfs/go-ipfs/repo" + + levelds "gx/ipfs/QmPdvXuXWAR6gtxxqZw42RtSADMwz4ijVmYHGS542b6cMz/go-ds-leveldb" + measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure" + flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs" + ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" + mount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount" + ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" +) + +func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) { + switch params["type"] { + case "mount": + mounts, ok := params["mounts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("mounts field wasnt an array") + } + + return r.openMountDatastore(mounts) + case "flatfs": + return r.openFlatfsDatastore(params) + case "mem": + return ds.NewMapDatastore(), nil + case "log": + child, err := r.constructDatastore(params["child"].(map[string]interface{})) + if err != nil { + return nil, err + } + + return ds.NewLogDatastore(child, params["name"].(string)), nil + case "measure": + child, err := r.constructDatastore(params["child"].(map[string]interface{})) + if err != nil { + return nil, err + } + + prefix := params["prefix"].(string) + + return r.openMeasureDB(prefix, child) + + case "levelds": + return r.openLeveldbDatastore(params) + default: + return nil, fmt.Errorf("unknown datastore type: %s", params["type"]) + } +} + +type mountConfig struct { + Path string + ChildType string + Child *json.RawMessage +} + +func (r *FSRepo) openMountDatastore(mountcfg []interface{}) (repo.Datastore, error) { + var mounts []mount.Mount + for _, iface := range mountcfg { + cfg := iface.(map[string]interface{}) + + child, err := r.constructDatastore(cfg) + if err != nil { + return nil, err + } + + prefix, found := cfg["mountpoint"] + if !found { + return nil, fmt.Errorf("no 'mountpoint' on mount") + } + + mounts = append(mounts, mount.Mount{ + Datastore: child, + Prefix: ds.NewKey(prefix.(string)), + }) + } + + return mount.New(mounts), nil +} + +func (r *FSRepo) openFlatfsDatastore(params map[string]interface{}) (repo.Datastore, error) { + p := params["path"].(string) + if !filepath.IsAbs(p) { + p = filepath.Join(r.path, p) + } + + sshardFun := params["shardFunc"].(string) + shardFun, err := flatfs.ParseShardFunc(sshardFun) + if err != nil { + return nil, err + } + + return flatfs.CreateOrOpen(p, shardFun, params["nosync"].(bool)) +} + +func (r *FSRepo) openLeveldbDatastore(params map[string]interface{}) (repo.Datastore, error) { + p := params["path"].(string) + if !filepath.IsAbs(p) { + p = filepath.Join(r.path, p) + } + + var c ldbopts.Compression + switch params["compression"].(string) { + case "none": + c = ldbopts.NoCompression + case "snappy": + c = ldbopts.SnappyCompression + case "": + fallthrough + default: + c = ldbopts.DefaultCompression + } + return levelds.NewDatastore(p, &levelds.Options{ + Compression: c, + }) +} + +func (r *FSRepo) openMeasureDB(prefix string, child repo.Datastore) (repo.Datastore, error) { + return measure.New(prefix, child), nil +} diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 34cb22b32..629d232ea 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -361,15 +361,20 @@ func (r *FSRepo) openKeystore() error { // openDatastore returns an error if the config file is not present. func (r *FSRepo) openDatastore() error { - switch r.config.Datastore.Type { - case "default", "leveldb", "": + if r.config.Datastore.Spec != nil { + d, err := r.constructDatastore(r.config.Datastore.Spec) + if err != nil { + return err + } + + r.ds = d + } else { + // TODO: This is for legacy configs, remove in the future d, err := openDefaultDatastore(r) if err != nil { return err } r.ds = d - default: - return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type) } // Wrap it with metrics gathering