mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-25 20:37:53 +08:00
make datastore configuration nicely customizable
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> make things super customizable License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> better json format License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> Migrate to new flatfs License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
This commit is contained in:
Jeromy
parent
c920033757
commit
7875674cae
@ -1,40 +1,43 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// DefaultDataStoreDirectory is the directory to store all the local IPFS data.
|
||||
const DefaultDataStoreDirectory = "datastore"
|
||||
|
||||
// Datastore tracks the configuration of the datastore.
|
||||
type Datastore struct {
|
||||
Type string
|
||||
Path string
|
||||
StorageMax string // in B, kB, kiB, MB, ...
|
||||
StorageGCWatermark int64 // in percentage to multiply on StorageMax
|
||||
GCPeriod string // in ns, us, ms, s, m, h
|
||||
Path string
|
||||
NoSync bool // deprecated
|
||||
|
||||
Spec map[string]interface{}
|
||||
|
||||
Params *json.RawMessage
|
||||
NoSync bool
|
||||
HashOnRead bool
|
||||
BloomFilterSize int
|
||||
}
|
||||
|
||||
func (d *Datastore) ParamData() []byte {
|
||||
if d.Params == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return []byte(*d.Params)
|
||||
}
|
||||
|
||||
type S3Datastore struct {
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
ACL string `json:"acl"`
|
||||
}
|
||||
|
||||
type FlatDS struct {
|
||||
Path string
|
||||
ShardFunc string
|
||||
Sync bool
|
||||
}
|
||||
|
||||
type LevelDB struct {
|
||||
Path string
|
||||
Compression string
|
||||
}
|
||||
|
||||
type SbsDS struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
// DataStorePath returns the default data store path given a configuration root
|
||||
// (set an empty string to have the default configuration root)
|
||||
func DataStorePath(configroot string) (string, error) {
|
||||
|
||||
@ -42,7 +42,7 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
|
||||
Gateway: "/ip4/127.0.0.1/tcp/8080",
|
||||
},
|
||||
|
||||
Datastore: datastore,
|
||||
Datastore: *datastore,
|
||||
Bootstrap: BootstrapPeerStrings(bootstrapPeers),
|
||||
Identity: identity,
|
||||
Discovery: Discovery{MDNS{
|
||||
@ -79,19 +79,38 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
func datastoreConfig() (Datastore, error) {
|
||||
dspath, err := DataStorePath("")
|
||||
if err != nil {
|
||||
return Datastore{}, err
|
||||
}
|
||||
return Datastore{
|
||||
Path: dspath,
|
||||
Type: "leveldb",
|
||||
func datastoreConfig() (*Datastore, error) {
|
||||
return &Datastore{
|
||||
StorageMax: "10GB",
|
||||
StorageGCWatermark: 90, // 90%
|
||||
GCPeriod: "1h",
|
||||
HashOnRead: false,
|
||||
BloomFilterSize: 0,
|
||||
Spec: map[string]interface{}{
|
||||
"type": "mount",
|
||||
"mounts": []interface{}{
|
||||
map[string]interface{}{
|
||||
"mountpoint": "/blocks",
|
||||
"type": "measure",
|
||||
"prefix": "flatfs.datastore",
|
||||
"child": map[string]interface{}{
|
||||
"type": "flatfs",
|
||||
"path": "blocks",
|
||||
"nosync": false,
|
||||
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"mountpoint": "/",
|
||||
"type": "measure",
|
||||
"prefix": "leveldb.datastore",
|
||||
"child": map[string]interface{}{
|
||||
"type": "levelds",
|
||||
"path": "datastore",
|
||||
"compression": "none",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
124
repo/fsrepo/datastores.go
Normal file
124
repo/fsrepo/datastores.go
Normal file
@ -0,0 +1,124 @@
|
||||
package fsrepo
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
repo "github.com/ipfs/go-ipfs/repo"
|
||||
|
||||
levelds "gx/ipfs/QmPdvXuXWAR6gtxxqZw42RtSADMwz4ijVmYHGS542b6cMz/go-ds-leveldb"
|
||||
measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure"
|
||||
flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs"
|
||||
ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
|
||||
mount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount"
|
||||
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) {
|
||||
switch params["type"] {
|
||||
case "mount":
|
||||
mounts, ok := params["mounts"].([]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("mounts field wasnt an array")
|
||||
}
|
||||
|
||||
return r.openMountDatastore(mounts)
|
||||
case "flatfs":
|
||||
return r.openFlatfsDatastore(params)
|
||||
case "mem":
|
||||
return ds.NewMapDatastore(), nil
|
||||
case "log":
|
||||
child, err := r.constructDatastore(params["child"].(map[string]interface{}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ds.NewLogDatastore(child, params["name"].(string)), nil
|
||||
case "measure":
|
||||
child, err := r.constructDatastore(params["child"].(map[string]interface{}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefix := params["prefix"].(string)
|
||||
|
||||
return r.openMeasureDB(prefix, child)
|
||||
|
||||
case "levelds":
|
||||
return r.openLeveldbDatastore(params)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown datastore type: %s", params["type"])
|
||||
}
|
||||
}
|
||||
|
||||
type mountConfig struct {
|
||||
Path string
|
||||
ChildType string
|
||||
Child *json.RawMessage
|
||||
}
|
||||
|
||||
func (r *FSRepo) openMountDatastore(mountcfg []interface{}) (repo.Datastore, error) {
|
||||
var mounts []mount.Mount
|
||||
for _, iface := range mountcfg {
|
||||
cfg := iface.(map[string]interface{})
|
||||
|
||||
child, err := r.constructDatastore(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefix, found := cfg["mountpoint"]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no 'mountpoint' on mount")
|
||||
}
|
||||
|
||||
mounts = append(mounts, mount.Mount{
|
||||
Datastore: child,
|
||||
Prefix: ds.NewKey(prefix.(string)),
|
||||
})
|
||||
}
|
||||
|
||||
return mount.New(mounts), nil
|
||||
}
|
||||
|
||||
func (r *FSRepo) openFlatfsDatastore(params map[string]interface{}) (repo.Datastore, error) {
|
||||
p := params["path"].(string)
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(r.path, p)
|
||||
}
|
||||
|
||||
sshardFun := params["shardFunc"].(string)
|
||||
shardFun, err := flatfs.ParseShardFunc(sshardFun)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return flatfs.CreateOrOpen(p, shardFun, params["nosync"].(bool))
|
||||
}
|
||||
|
||||
func (r *FSRepo) openLeveldbDatastore(params map[string]interface{}) (repo.Datastore, error) {
|
||||
p := params["path"].(string)
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(r.path, p)
|
||||
}
|
||||
|
||||
var c ldbopts.Compression
|
||||
switch params["compression"].(string) {
|
||||
case "none":
|
||||
c = ldbopts.NoCompression
|
||||
case "snappy":
|
||||
c = ldbopts.SnappyCompression
|
||||
case "":
|
||||
fallthrough
|
||||
default:
|
||||
c = ldbopts.DefaultCompression
|
||||
}
|
||||
return levelds.NewDatastore(p, &levelds.Options{
|
||||
Compression: c,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *FSRepo) openMeasureDB(prefix string, child repo.Datastore) (repo.Datastore, error) {
|
||||
return measure.New(prefix, child), nil
|
||||
}
|
||||
@ -361,15 +361,20 @@ func (r *FSRepo) openKeystore() error {
|
||||
|
||||
// openDatastore returns an error if the config file is not present.
|
||||
func (r *FSRepo) openDatastore() error {
|
||||
switch r.config.Datastore.Type {
|
||||
case "default", "leveldb", "":
|
||||
if r.config.Datastore.Spec != nil {
|
||||
d, err := r.constructDatastore(r.config.Datastore.Spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.ds = d
|
||||
} else {
|
||||
// TODO: This is for legacy configs, remove in the future
|
||||
d, err := openDefaultDatastore(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.ds = d
|
||||
default:
|
||||
return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type)
|
||||
}
|
||||
|
||||
// Wrap it with metrics gathering
|
||||
|
||||
Loading…
Reference in New Issue
Block a user