diff --git a/repo/fsrepo/config_test.go b/repo/fsrepo/config_test.go index 1c62241a2..79fda4c85 100644 --- a/repo/fsrepo/config_test.go +++ b/repo/fsrepo/config_test.go @@ -87,9 +87,9 @@ func TestDefaultDatastoreConfig(t *testing.T) { t.Fatal(err) } - expected := "/blocks:{flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2};/:{levelds;datastore};" - if dsc.DiskId() != expected { - t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId()) + expected := `{"mounts":[{"mountpoint":{"string":"/blocks"},"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":{"string":"/"},"path":"datastore","type":"levelds"}],"type":"mount"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) @@ -125,9 +125,9 @@ func TestLevelDbConfig(t *testing.T) { t.Fatal(err) } - expected := "levelds;datastore" - if dsc.DiskId() != expected { - t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId()) + expected := `{"path":"datastore","type":"levelds"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) @@ -163,9 +163,9 @@ func TestFlatfsConfig(t *testing.T) { t.Fatal(err) } - expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2" - if dsc.DiskId() != expected { - t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId()) + expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) @@ -201,9 +201,9 @@ func TestMeasureConfig(t *testing.T) { t.Fatal(err) } - expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2" - if dsc.DiskId() != expected { - t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId()) + expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) diff --git a/repo/fsrepo/datastores.go b/repo/fsrepo/datastores.go index 807097ac3..c5b81ce15 100644 --- a/repo/fsrepo/datastores.go +++ b/repo/fsrepo/datastores.go @@ -2,6 +2,7 @@ package fsrepo import ( "bytes" + "encoding/json" "fmt" "path/filepath" @@ -18,17 +19,28 @@ import ( // ConfigFromMap creates a new datastore config from a map type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error) +type DiskSpec map[string]interface{} + type DatastoreConfig interface { - // DiskId is a unique id representing the Datastore config as - // stored on disk, runtime config values are not part of this Id. - // Returns an empty string if the datastore does not have an on - // disk representation. No length limit. - DiskId() string + // DiskSpec returns a minimal configuration of the datastore + // represting what is stored on disk. Run time values are + // excluded. + DiskSpec() DiskSpec // Create instantiate a new datastore from this config Create(path string) (repo.Datastore, error) } +func (spec DiskSpec) String() string { + b, err := json.Marshal(spec) + if err != nil { + // should not happen + panic(err) + } + b = bytes.TrimSpace(b) + return string(b) +} + var datastores map[string]ConfigFromMap func init() { @@ -94,12 +106,19 @@ func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error return &res, nil } -func (c *mountDatastoreConfig) DiskId() string { - buf := new(bytes.Buffer) - for _, m := range c.mounts { - fmt.Fprintf(buf, "%s:{%s};", m.prefix.String(), m.ds.DiskId()) +func (c *mountDatastoreConfig) DiskSpec() DiskSpec { + cfg := map[string]interface{}{"type": "mount"} + mounts := make([]interface{}, len(c.mounts)) + for i, m := range c.mounts { + c := m.ds.DiskSpec() + if c == nil { + c = make(map[string]interface{}) + } + c["mountpoint"] = m.prefix + mounts[i] = c } - return buf.String() + cfg["mounts"] = mounts + return cfg } func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) { @@ -147,8 +166,12 @@ func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, erro return &c, nil } -func (c *flatfsDatastoreConfig) DiskId() string { - return fmt.Sprintf("flatfs;%s;%s", c.path, c.shardFun.String()) +func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec { + return map[string]interface{}{ + "type": "flatfs", + "path": c.path, + "shardFunc": c.shardFun.String(), + } } func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) { @@ -188,8 +211,11 @@ func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err return &c, nil } -func (c *leveldsDatastoreConfig) DiskId() string { - return fmt.Sprintf("levelds;%s", c.path) +func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec { + return map[string]interface{}{ + "type": "levelds", + "path": c.path, + } } func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) { @@ -211,8 +237,8 @@ func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) return &memDatastoreConfig{params}, nil } -func (c *memDatastoreConfig) DiskId() string { - return "" +func (c *memDatastoreConfig) DiskSpec() DiskSpec { + return nil } func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) { @@ -249,8 +275,8 @@ func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) { return ds.NewLogDatastore(child, c.name), nil } -func (c *logDatastoreConfig) DiskId() string { - return c.child.DiskId() +func (c *logDatastoreConfig) DiskSpec() DiskSpec { + return c.child.DiskSpec() } type measureDatastoreConfig struct { @@ -274,8 +300,8 @@ func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err return &measureDatastoreConfig{child, prefix}, nil } -func (c *measureDatastoreConfig) DiskId() string { - return c.child.DiskId() +func (c *measureDatastoreConfig) DiskSpec() DiskSpec { + return c.child.DiskSpec() } func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) { diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 526464776..44ef42382 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -1,7 +1,6 @@ package fsrepo import ( - "encoding/json" "errors" "fmt" "io" @@ -368,16 +367,16 @@ func (r *FSRepo) openDatastore() error { if err != nil { return err } - diskId := dsc.DiskId() + spec := dsc.DiskSpec() - oldId, _, err := r.readSpec() + oldSpec, err := r.readSpec() if err == nil { - if oldId != diskId { + if oldSpec != spec.String() { return fmt.Errorf("Datastore configuration of '%s' does not match what is on disk '%s'", - oldId, diskId) + oldSpec, spec.String()) } } else if os.IsNotExist(err) { - err := r.writeSpec(diskId, r.config.Datastore.Spec) + err := r.writeSpec(spec.String()) if err != nil { return err } @@ -398,55 +397,27 @@ func (r *FSRepo) openDatastore() error { return nil } -var SpecFn = "spec" +var SpecFn = "datastore_spec" -func (r *FSRepo) readSpec() (string, map[string]interface{}, error) { +func (r *FSRepo) readSpec() (string, error) { fn, err := config.Path(r.path, SpecFn) if err != nil { - return "", nil, err + return "", err } b, err := ioutil.ReadFile(fn) if err != nil { - return "", nil, err + return "", err } - idspec := make(map[string]interface{}) - err = json.Unmarshal(b, &idspec) - if err != nil { - return "", nil, err - } - id, ok := idspec["id"].(string) - if !ok { - return "", nil, fmt.Errorf("could not retrieve 'id' field from spec file") - } - spec, ok := idspec["spec"].(map[string]interface{}) - if !ok { - return "", nil, fmt.Errorf("could not retrieve 'spec' field from spec file") - } - - dsc, err := AnyDatastoreConfig(spec) - if err != nil { - return "", nil, err - } - computedId := dsc.DiskId() - if computedId != id { - return "", nil, fmt.Errorf("bad spec file, computed id (%s) does not match given (%s)", - computedId, id) - } - - return id, spec, nil + return strings.TrimSpace(string(b)), nil } -func (r *FSRepo) writeSpec(id string, spec map[string]interface{}) error { +func (r *FSRepo) writeSpec(spec string) error { fn, err := config.Path(r.path, SpecFn) if err != nil { return err } - idspec := map[string]interface{}{ - "id": id, - "spec": spec, - } - b, err := json.Marshal(idspec) - err = ioutil.WriteFile(fn, b, 0666) + b := []byte(spec) + err = ioutil.WriteFile(fn, b, 0600) if err != nil { return err }