kubo/repo/fsrepo/datastores.go

package fsrepo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sort"

	"github.com/ipfs/kubo/repo"

	ds "github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/mount"
	dssync "github.com/ipfs/go-datastore/sync"
	"github.com/ipfs/go-ds-measure"
)

// ConfigFromMap creates a new datastore config from a map.
type ConfigFromMap func(map[string]any) (DatastoreConfig, error)

// DatastoreConfig is an abstraction of a datastore config. A "spec" is first
// converted to a DatastoreConfig and then Create() is called to instantiate a
// new datastore.
type DatastoreConfig interface {
	// DiskSpec returns a minimal configuration of the datastore representing
	// what is stored on disk. Run time values are excluded.
	DiskSpec() DiskSpec

	// Create instantiates a new datastore from this config.
	Create(path string) (repo.Datastore, error)
}

// DiskSpec is a minimal representation of the characteristic values of the
// datastore. If two diskspecs are the same, the loader assumes that they refer
// to exactly the same datastore. If they differ at all, it is assumed they are
// completely different datastores and a migration will be performed. Runtime
// values such as cache options or concurrency options should not be added
// here.
type DiskSpec map[string]any

// Bytes returns a minimal JSON encoding of the DiskSpec.
func (spec DiskSpec) Bytes() []byte {
	b, err := json.Marshal(spec)
	if err != nil {
		// should not happen
		panic(err)
	}
	return bytes.TrimSpace(b)
}

// String returns a minimal JSON encoding of the DiskSpec.
func (spec DiskSpec) String() string {
	return string(spec.Bytes())
}

var datastores map[string]ConfigFromMap

func init() {
	datastores = map[string]ConfigFromMap{
		"mount":   MountDatastoreConfig,
		"mem":     MemDatastoreConfig,
		"log":     LogDatastoreConfig,
		"measure": MeasureDatastoreConfig,
	}
}

func AddDatastoreConfigHandler(name string, dsc ConfigFromMap) error {
	_, ok := datastores[name]
	if ok {
		return fmt.Errorf("already have a datastore named %q", name)
	}

	datastores[name] = dsc
	return nil
}

// AnyDatastoreConfig returns a DatastoreConfig from a spec based on
// the "type" parameter.
func AnyDatastoreConfig(params map[string]any) (DatastoreConfig, error) {
	which, ok := params["type"].(string)
	if !ok {
		return nil, fmt.Errorf("'type' field missing or not a string")
	}
	fun, ok := datastores[which]
	if !ok {
		return nil, fmt.Errorf("unknown datastore type: %s", which)
	}
	return fun(params)
}

type mountDatastoreConfig struct {
	mounts []premount
}

type premount struct {
	ds     DatastoreConfig
	prefix ds.Key
}

// MountDatastoreConfig returns a mount DatastoreConfig from a spec.
func MountDatastoreConfig(params map[string]any) (DatastoreConfig, error) {
	var res mountDatastoreConfig
	mounts, ok := params["mounts"].([]any)
	if !ok {
		return nil, fmt.Errorf("'mounts' field is missing or not an array")
	}
	for _, iface := range mounts {
		cfg, ok := iface.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("expected map for mountpoint")
		}

		child, err := AnyDatastoreConfig(cfg)
		if err != nil {
			return nil, err
		}

		prefix, found := cfg["mountpoint"]
		if !found {
			return nil, fmt.Errorf("no 'mountpoint' on mount")
		}

		res.mounts = append(res.mounts, premount{
			ds:     child,
			prefix: ds.NewKey(prefix.(string)),
		})
	}
	sort.Slice(res.mounts,
		func(i, j int) bool {
			return res.mounts[i].prefix.String() > res.mounts[j].prefix.String()
		})

	return &res, nil
}

func (c *mountDatastoreConfig) DiskSpec() DiskSpec {
	cfg := map[string]any{"type": "mount"}
	mounts := make([]any, len(c.mounts))
	for i, m := range c.mounts {
		c := m.ds.DiskSpec()
		if c == nil {
			c = make(map[string]any)
		}
		c["mountpoint"] = m.prefix.String()
		mounts[i] = c
	}
	cfg["mounts"] = mounts
	return cfg
}

func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
	mounts := make([]mount.Mount, len(c.mounts))
	for i, m := range c.mounts {
		ds, err := m.ds.Create(path)
		if err != nil {
			return nil, err
		}
		mounts[i].Datastore = ds
		mounts[i].Prefix = m.prefix
	}
	return mount.New(mounts), nil
}

type memDatastoreConfig struct {
	cfg map[string]any
}

// MemDatastoreConfig returns a memory DatastoreConfig from a spec.
func MemDatastoreConfig(params map[string]any) (DatastoreConfig, error) {
	return &memDatastoreConfig{params}, nil
}

func (c *memDatastoreConfig) DiskSpec() DiskSpec {
	return nil
}

func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
	return dssync.MutexWrap(ds.NewMapDatastore()), nil
}

type logDatastoreConfig struct {
	child DatastoreConfig
	name  string
}

// LogDatastoreConfig returns a log DatastoreConfig from a spec.
func LogDatastoreConfig(params map[string]any) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]any)
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	name, ok := params["name"].(string)
	if !ok {
		return nil, fmt.Errorf("'name' field was missing or not a string")
	}
	return &logDatastoreConfig{child, name}, nil
}

func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return ds.NewLogDatastore(child, c.name), nil
}

func (c *logDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

type measureDatastoreConfig struct {
	child  DatastoreConfig
	prefix string
}

// MeasureDatastoreConfig returns a measure DatastoreConfig from a spec.
func MeasureDatastoreConfig(params map[string]any) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]any)
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	prefix, ok := params["prefix"].(string)
	if !ok {
		return nil, fmt.Errorf("'prefix' field was missing or not a string")
	}
	return &measureDatastoreConfig{child, prefix}, nil
}

func (c *measureDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return measure.New(c.prefix, child), nil
}