From 39a87a0fb01e36d682f9b1bfcd3bf46c2ea2a00d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 20 Aug 2018 15:28:14 +0200 Subject: [PATCH] Extract /mfs The /mfs module has been extracted to github.com/ipfs/go-mfs All history has been retained in the new repository. README, LICENSE, Makefiles and CI integration have been added to the new location. License: MIT Signed-off-by: Hector Sanjuan --- core/commands/add.go | 2 +- core/commands/files.go | 2 +- core/core.go | 2 +- core/corerepo/gc.go | 2 +- core/coreunix/add.go | 2 +- fuse/ipns/ipns_unix.go | 2 +- mfs/dir.go | 462 ---------------- mfs/fd.go | 151 ----- mfs/file.go | 146 ----- mfs/mfs_test.go | 1184 ---------------------------------------- mfs/ops.go | 223 -------- mfs/repub_test.go | 77 --- mfs/system.go | 292 ---------- package.json | 7 + 14 files changed, 13 insertions(+), 2541 deletions(-) delete mode 100644 mfs/dir.go delete mode 100644 mfs/fd.go delete mode 100644 mfs/file.go delete mode 100644 mfs/mfs_test.go delete mode 100644 mfs/ops.go delete mode 100644 mfs/repub_test.go delete mode 100644 mfs/system.go diff --git a/core/commands/add.go b/core/commands/add.go index fd358d381..a7eb64388 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -10,7 +10,6 @@ import ( core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreunix" filestore "github.com/ipfs/go-ipfs/filestore" - mfs "github.com/ipfs/go-ipfs/mfs" dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" dagtest "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag/test" blockservice "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice" @@ -20,6 +19,7 @@ import ( files "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit/files" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" cmds "gx/ipfs/QmUQb3xtNzkQCgTj2NjaqcJZNv2nfSSub2QAdy9DtQMRBT/go-ipfs-cmds" offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline" bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore" diff --git a/core/commands/files.go b/core/commands/files.go index ffa9eee60..db21392d7 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -15,7 +15,6 @@ import ( lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - mfs "github.com/ipfs/go-ipfs/mfs" dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" bservice "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice" path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path" @@ -26,6 +25,7 @@ import ( humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" cmdkit "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" cmds "gx/ipfs/QmUQb3xtNzkQCgTj2NjaqcJZNv2nfSSub2QAdy9DtQMRBT/go-ipfs-cmds" offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline" diff --git a/core/core.go b/core/core.go index d5944d106..492244a4b 100644 --- a/core/core.go +++ b/core/core.go @@ -24,7 +24,6 @@ import ( rp "github.com/ipfs/go-ipfs/exchange/reprovide" filestore "github.com/ipfs/go-ipfs/filestore" mount "github.com/ipfs/go-ipfs/fuse/mount" - mfs "github.com/ipfs/go-ipfs/mfs" namesys "github.com/ipfs/go-ipfs/namesys" ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" p2p "github.com/ipfs/go-ipfs/p2p" @@ -36,6 +35,7 @@ import ( ic "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto" p2phost "gx/ipfs/QmQ1hwb95uSSZR8jSPJysnfHxBDQAykSXsmz5TwTzxjq2Z/go-libp2p-host" config "gx/ipfs/QmQSG7YCizeUH2bWatzp6uK9Vm3m7LA5jpxGa9QqgpNKw4/go-ipfs-config" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" bitswap "gx/ipfs/QmQk1Rqy5XSBzXykMSsgiXfnhivCSnFpykx4M2j6DD1nBH/go-bitswap" bsnet "gx/ipfs/QmQk1Rqy5XSBzXykMSsgiXfnhivCSnFpykx4M2j6DD1nBH/go-bitswap/network" merkledag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index b1cfff630..375bb0a64 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -7,11 +7,11 @@ import ( "time" "github.com/ipfs/go-ipfs/core" - mfs "github.com/ipfs/go-ipfs/mfs" gc "github.com/ipfs/go-ipfs/pin/gc" repo "github.com/ipfs/go-ipfs/repo" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" ) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 311bf50a4..2cc1af543 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -11,7 +11,6 @@ import ( "strconv" core "github.com/ipfs/go-ipfs/core" - mfs "github.com/ipfs/go-ipfs/mfs" "github.com/ipfs/go-ipfs/pin" dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" unixfs "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" @@ -20,6 +19,7 @@ import ( trickle "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/importer/trickle" files "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit/files" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker" bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index c39dd8497..d26ddc6b8 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -12,13 +12,13 @@ import ( "os" core "github.com/ipfs/go-ipfs/core" - mfs "github.com/ipfs/go-ipfs/mfs" namesys "github.com/ipfs/go-ipfs/namesys" dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path" ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" ci "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto" + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" fuse "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse" fs "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs" diff --git a/mfs/dir.go b/mfs/dir.go deleted file mode 100644 index 50d3747c7..000000000 --- a/mfs/dir.go +++ /dev/null @@ -1,462 +0,0 @@ -package mfs - -import ( - "context" - "errors" - "fmt" - "os" - "path" - "sync" - "time" - - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" - uio "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/io" - ufspb "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/pb" - - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format" -) - -var ErrNotYetImplemented = errors.New("not yet implemented") -var ErrInvalidChild = errors.New("invalid child node") -var ErrDirExists = errors.New("directory already has entry by that name") - -type Directory struct { - dserv ipld.DAGService - parent childCloser - - childDirs map[string]*Directory - files map[string]*File - - lock sync.Mutex - ctx context.Context - - // UnixFS directory implementation used for creating, - // reading and editing directories. - unixfsDir uio.Directory - - modTime time.Time - - name string -} - -// NewDirectory constructs a new MFS directory. -// -// You probably don't want to call this directly. Instead, construct a new root -// using NewRoot. -func NewDirectory(ctx context.Context, name string, node ipld.Node, parent childCloser, dserv ipld.DAGService) (*Directory, error) { - db, err := uio.NewDirectoryFromNode(dserv, node) - if err != nil { - return nil, err - } - - return &Directory{ - dserv: dserv, - ctx: ctx, - name: name, - unixfsDir: db, - parent: parent, - childDirs: make(map[string]*Directory), - files: make(map[string]*File), - modTime: time.Now(), - }, nil -} - -// GetCidBuilder gets the CID builder of the root node -func (d *Directory) GetCidBuilder() cid.Builder { - return d.unixfsDir.GetCidBuilder() -} - -// SetCidBuilder sets the CID builder -func (d *Directory) SetCidBuilder(b cid.Builder) { - d.unixfsDir.SetCidBuilder(b) -} - -// closeChild updates the child by the given name to the dag node 'nd' -// and changes its own dag node -func (d *Directory) closeChild(name string, nd ipld.Node, sync bool) error { - mynd, err := d.closeChildUpdate(name, nd, sync) - if err != nil { - return err - } - - if sync { - return d.parent.closeChild(d.name, mynd, true) - } - return nil -} - -// closeChildUpdate is the portion of closeChild that needs to be locked around -func (d *Directory) closeChildUpdate(name string, nd ipld.Node, sync bool) (*dag.ProtoNode, error) { - d.lock.Lock() - defer d.lock.Unlock() - - err := d.updateChild(name, nd) - if err != nil { - return nil, err - } - - if sync { - return d.flushCurrentNode() - } - return nil, nil -} - -func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) { - nd, err := d.unixfsDir.GetNode() - if err != nil { - return nil, err - } - - err = d.dserv.Add(d.ctx, nd) - if err != nil { - return nil, err - } - - pbnd, ok := nd.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - return pbnd.Copy().(*dag.ProtoNode), nil -} - -func (d *Directory) updateChild(name string, nd ipld.Node) error { - err := d.AddUnixFSChild(name, nd) - if err != nil { - return err - } - - d.modTime = time.Now() - - return nil -} - -func (d *Directory) Type() NodeType { - return TDir -} - -// childNode returns a FSNode under this directory by the given name if it exists. -// it does *not* check the cached dirs and files -func (d *Directory) childNode(name string) (FSNode, error) { - nd, err := d.childFromDag(name) - if err != nil { - return nil, err - } - - return d.cacheNode(name, nd) -} - -// cacheNode caches a node into d.childDirs or d.files and returns the FSNode. -func (d *Directory) cacheNode(name string, nd ipld.Node) (FSNode, error) { - switch nd := nd.(type) { - case *dag.ProtoNode: - i, err := ft.FromBytes(nd.Data()) - if err != nil { - return nil, err - } - - switch i.GetType() { - case ufspb.Data_Directory, ufspb.Data_HAMTShard: - ndir, err := NewDirectory(d.ctx, name, nd, d, d.dserv) - if err != nil { - return nil, err - } - - d.childDirs[name] = ndir - return ndir, nil - case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink: - nfi, err := NewFile(name, nd, d, d.dserv) - if err != nil { - return nil, err - } - d.files[name] = nfi - return nfi, nil - case ufspb.Data_Metadata: - return nil, ErrNotYetImplemented - default: - return nil, ErrInvalidChild - } - case *dag.RawNode: - nfi, err := NewFile(name, nd, d, d.dserv) - if err != nil { - return nil, err - } - d.files[name] = nfi - return nfi, nil - default: - return nil, fmt.Errorf("unrecognized node type in cache node") - } -} - -// Child returns the child of this directory by the given name -func (d *Directory) Child(name string) (FSNode, error) { - d.lock.Lock() - defer d.lock.Unlock() - return d.childUnsync(name) -} - -func (d *Directory) Uncache(name string) { - d.lock.Lock() - defer d.lock.Unlock() - delete(d.files, name) - delete(d.childDirs, name) -} - -// childFromDag searches through this directories dag node for a child link -// with the given name -func (d *Directory) childFromDag(name string) (ipld.Node, error) { - return d.unixfsDir.Find(d.ctx, name) -} - -// childUnsync returns the child under this directory by the given name -// without locking, useful for operations which already hold a lock -func (d *Directory) childUnsync(name string) (FSNode, error) { - cdir, ok := d.childDirs[name] - if ok { - return cdir, nil - } - - cfile, ok := d.files[name] - if ok { - return cfile, nil - } - - return d.childNode(name) -} - -type NodeListing struct { - Name string - Type int - Size int64 - Hash string -} - -func (d *Directory) ListNames(ctx context.Context) ([]string, error) { - d.lock.Lock() - defer d.lock.Unlock() - - var out []string - err := d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error { - out = append(out, l.Name) - return nil - }) - if err != nil { - return nil, err - } - - return out, nil -} - -func (d *Directory) List(ctx context.Context) ([]NodeListing, error) { - var out []NodeListing - err := d.ForEachEntry(ctx, func(nl NodeListing) error { - out = append(out, nl) - return nil - }) - return out, err -} - -func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) error { - d.lock.Lock() - defer d.lock.Unlock() - return d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error { - c, err := d.childUnsync(l.Name) - if err != nil { - return err - } - - nd, err := c.GetNode() - if err != nil { - return err - } - - child := NodeListing{ - Name: l.Name, - Type: int(c.Type()), - Hash: nd.Cid().String(), - } - - if c, ok := c.(*File); ok { - size, err := c.Size() - if err != nil { - return err - } - child.Size = size - } - - return f(child) - }) -} - -func (d *Directory) Mkdir(name string) (*Directory, error) { - d.lock.Lock() - defer d.lock.Unlock() - - fsn, err := d.childUnsync(name) - if err == nil { - switch fsn := fsn.(type) { - case *Directory: - return fsn, os.ErrExist - case *File: - return nil, os.ErrExist - default: - return nil, fmt.Errorf("unrecognized type: %#v", fsn) - } - } - - ndir := ft.EmptyDirNode() - ndir.SetCidBuilder(d.GetCidBuilder()) - - err = d.dserv.Add(d.ctx, ndir) - if err != nil { - return nil, err - } - - err = d.AddUnixFSChild(name, ndir) - if err != nil { - return nil, err - } - - dirobj, err := NewDirectory(d.ctx, name, ndir, d, d.dserv) - if err != nil { - return nil, err - } - - d.childDirs[name] = dirobj - return dirobj, nil -} - -func (d *Directory) Unlink(name string) error { - d.lock.Lock() - defer d.lock.Unlock() - - delete(d.childDirs, name) - delete(d.files, name) - - return d.unixfsDir.RemoveChild(d.ctx, name) -} - -func (d *Directory) Flush() error { - nd, err := d.GetNode() - if err != nil { - return err - } - - return d.parent.closeChild(d.name, nd, true) -} - -// AddChild adds the node 'nd' under this directory giving it the name 'name' -func (d *Directory) AddChild(name string, nd ipld.Node) error { - d.lock.Lock() - defer d.lock.Unlock() - - _, err := d.childUnsync(name) - if err == nil { - return ErrDirExists - } - - err = d.dserv.Add(d.ctx, nd) - if err != nil { - return err - } - - err = d.AddUnixFSChild(name, nd) - if err != nil { - return err - } - - d.modTime = time.Now() - return nil -} - -// AddUnixFSChild adds a child to the inner UnixFS directory -// and transitions to a HAMT implementation if needed. -func (d *Directory) AddUnixFSChild(name string, node ipld.Node) error { - if uio.UseHAMTSharding { - // If the directory HAMT implementation is being used and this - // directory is actually a basic implementation switch it to HAMT. - if basicDir, ok := d.unixfsDir.(*uio.BasicDirectory); ok { - hamtDir, err := basicDir.SwitchToSharding(d.ctx) - if err != nil { - return err - } - d.unixfsDir = hamtDir - } - } - - err := d.unixfsDir.AddChild(d.ctx, name, node) - if err != nil { - return err - } - - return nil -} - -func (d *Directory) sync() error { - for name, dir := range d.childDirs { - nd, err := dir.GetNode() - if err != nil { - return err - } - - err = d.updateChild(name, nd) - if err != nil { - return err - } - } - - for name, file := range d.files { - nd, err := file.GetNode() - if err != nil { - return err - } - - err = d.updateChild(name, nd) - if err != nil { - return err - } - } - - return nil -} - -func (d *Directory) Path() string { - cur := d - var out string - for cur != nil { - switch parent := cur.parent.(type) { - case *Directory: - out = path.Join(cur.name, out) - cur = parent - case *Root: - return "/" + out - default: - panic("directory parent neither a directory nor a root") - } - } - return out -} - -func (d *Directory) GetNode() (ipld.Node, error) { - d.lock.Lock() - defer d.lock.Unlock() - - err := d.sync() - if err != nil { - return nil, err - } - - nd, err := d.unixfsDir.GetNode() - if err != nil { - return nil, err - } - - err = d.dserv.Add(d.ctx, nd) - if err != nil { - return nil, err - } - - return nd.Copy(), err -} diff --git a/mfs/fd.go b/mfs/fd.go deleted file mode 100644 index 8b84000fd..000000000 --- a/mfs/fd.go +++ /dev/null @@ -1,151 +0,0 @@ -package mfs - -import ( - "fmt" - "io" - - mod "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/mod" - - context "context" -) - -type FileDescriptor interface { - io.Reader - CtxReadFull(context.Context, []byte) (int, error) - - io.Writer - io.WriterAt - - io.Closer - io.Seeker - - Truncate(int64) error - Size() (int64, error) - Sync() error - Flush() error -} - -type fileDescriptor struct { - inode *File - mod *mod.DagModifier - perms int - sync bool - hasChanges bool - - closed bool -} - -// Size returns the size of the file referred to by this descriptor -func (fi *fileDescriptor) Size() (int64, error) { - return fi.mod.Size() -} - -// Truncate truncates the file to size -func (fi *fileDescriptor) Truncate(size int64) error { - if fi.perms == OpenReadOnly { - return fmt.Errorf("cannot call truncate on readonly file descriptor") - } - fi.hasChanges = true - return fi.mod.Truncate(size) -} - -// Write writes the given data to the file at its current offset -func (fi *fileDescriptor) Write(b []byte) (int, error) { - if fi.perms == OpenReadOnly { - return 0, fmt.Errorf("cannot write on not writeable descriptor") - } - fi.hasChanges = true - return fi.mod.Write(b) -} - -// Read reads into the given buffer from the current offset -func (fi *fileDescriptor) Read(b []byte) (int, error) { - if fi.perms == OpenWriteOnly { - return 0, fmt.Errorf("cannot read on write-only descriptor") - } - return fi.mod.Read(b) -} - -// Read reads into the given buffer from the current offset -func (fi *fileDescriptor) CtxReadFull(ctx context.Context, b []byte) (int, error) { - if fi.perms == OpenWriteOnly { - return 0, fmt.Errorf("cannot read on write-only descriptor") - } - return fi.mod.CtxReadFull(ctx, b) -} - -// Close flushes, then propogates the modified dag node up the directory structure -// and signals a republish to occur -func (fi *fileDescriptor) Close() error { - defer func() { - switch fi.perms { - case OpenReadOnly: - fi.inode.desclock.RUnlock() - case OpenWriteOnly, OpenReadWrite: - fi.inode.desclock.Unlock() - } - }() - - if fi.closed { - panic("attempted to close file descriptor twice!") - } - - if fi.hasChanges { - err := fi.mod.Sync() - if err != nil { - return err - } - - fi.hasChanges = false - - // explicitly stay locked for flushUp call, - // it will manage the lock for us - return fi.flushUp(fi.sync) - } - - return nil -} - -func (fi *fileDescriptor) Sync() error { - return fi.flushUp(false) -} - -func (fi *fileDescriptor) Flush() error { - return fi.flushUp(true) -} - -// flushUp syncs the file and adds it to the dagservice -// it *must* be called with the File's lock taken -func (fi *fileDescriptor) flushUp(fullsync bool) error { - nd, err := fi.mod.GetNode() - if err != nil { - return err - } - - err = fi.inode.dserv.Add(context.TODO(), nd) - if err != nil { - return err - } - - fi.inode.nodelk.Lock() - fi.inode.node = nd - name := fi.inode.name - parent := fi.inode.parent - fi.inode.nodelk.Unlock() - - return parent.closeChild(name, nd, fullsync) -} - -// Seek implements io.Seeker -func (fi *fileDescriptor) Seek(offset int64, whence int) (int64, error) { - return fi.mod.Seek(offset, whence) -} - -// Write At writes the given bytes at the offset 'at' -func (fi *fileDescriptor) WriteAt(b []byte, at int64) (int, error) { - if fi.perms == OpenReadOnly { - return 0, fmt.Errorf("cannot write on not writeable descriptor") - } - fi.hasChanges = true - return fi.mod.WriteAt(b, at) -} diff --git a/mfs/file.go b/mfs/file.go deleted file mode 100644 index fec9c2125..000000000 --- a/mfs/file.go +++ /dev/null @@ -1,146 +0,0 @@ -package mfs - -import ( - "context" - "fmt" - "sync" - - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" - mod "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/mod" - - chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker" - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format" -) - -type File struct { - parent childCloser - - name string - - desclock sync.RWMutex - - dserv ipld.DAGService - node ipld.Node - nodelk sync.Mutex - - RawLeaves bool -} - -// NewFile returns a NewFile object with the given parameters. If the -// Cid version is non-zero RawLeaves will be enabled. -func NewFile(name string, node ipld.Node, parent childCloser, dserv ipld.DAGService) (*File, error) { - fi := &File{ - dserv: dserv, - parent: parent, - name: name, - node: node, - } - if node.Cid().Prefix().Version > 0 { - fi.RawLeaves = true - } - return fi, nil -} - -const ( - OpenReadOnly = iota - OpenWriteOnly - OpenReadWrite -) - -func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { - fi.nodelk.Lock() - node := fi.node - fi.nodelk.Unlock() - - switch node := node.(type) { - case *dag.ProtoNode: - fsn, err := ft.FSNodeFromBytes(node.Data()) - if err != nil { - return nil, err - } - - switch fsn.Type() { - default: - return nil, fmt.Errorf("unsupported fsnode type for 'file'") - case ft.TSymlink: - return nil, fmt.Errorf("symlinks not yet supported") - case ft.TFile, ft.TRaw: - // OK case - } - case *dag.RawNode: - // Ok as well. - } - - switch flags { - case OpenReadOnly: - fi.desclock.RLock() - case OpenWriteOnly, OpenReadWrite: - fi.desclock.Lock() - default: - // TODO: support other modes - return nil, fmt.Errorf("mode not supported") - } - - dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunker.DefaultSplitter) - if err != nil { - return nil, err - } - dmod.RawLeaves = fi.RawLeaves - - return &fileDescriptor{ - inode: fi, - perms: flags, - sync: sync, - mod: dmod, - }, nil -} - -// Size returns the size of this file -func (fi *File) Size() (int64, error) { - fi.nodelk.Lock() - defer fi.nodelk.Unlock() - switch nd := fi.node.(type) { - case *dag.ProtoNode: - pbd, err := ft.FromBytes(nd.Data()) - if err != nil { - return 0, err - } - return int64(pbd.GetFilesize()), nil - case *dag.RawNode: - return int64(len(nd.RawData())), nil - default: - return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()") - } -} - -// GetNode returns the dag node associated with this file -func (fi *File) GetNode() (ipld.Node, error) { - fi.nodelk.Lock() - defer fi.nodelk.Unlock() - return fi.node, nil -} - -func (fi *File) Flush() error { - // open the file in fullsync mode - fd, err := fi.Open(OpenWriteOnly, true) - if err != nil { - return err - } - - defer fd.Close() - - return fd.Flush() -} - -func (fi *File) Sync() error { - // just being able to take the writelock means the descriptor is synced - fi.desclock.Lock() - fi.desclock.Unlock() - return nil -} - -// Type returns the type FSNode this is -func (fi *File) Type() NodeType { - return TFile -} diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go deleted file mode 100644 index 524732f2c..000000000 --- a/mfs/mfs_test.go +++ /dev/null @@ -1,1184 +0,0 @@ -package mfs - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "os" - "sort" - "sync" - "testing" - "time" - - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" - bserv "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice" - "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path" - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" - importer "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/importer" - uio "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/io" - - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" - ds "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore" - dssync "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore/sync" - offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline" - chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker" - bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore" - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format" -) - -func emptyDirNode() *dag.ProtoNode { - return dag.NodeWithData(ft.FolderPBData()) -} - -func getDagserv(t *testing.T) ipld.DAGService { - db := dssync.MutexWrap(ds.NewMapDatastore()) - bs := bstore.NewBlockstore(db) - blockserv := bserv.New(bs, offline.Exchange(bs)) - return dag.NewDAGService(blockserv) -} - -func getRandFile(t *testing.T, ds ipld.DAGService, size int64) ipld.Node { - r := io.LimitReader(u.NewTimeSeededRand(), size) - return fileNodeFromReader(t, ds, r) -} - -func fileNodeFromReader(t *testing.T, ds ipld.DAGService, r io.Reader) ipld.Node { - nd, err := importer.BuildDagFromReader(ds, chunker.DefaultSplitter(r)) - if err != nil { - t.Fatal(err) - } - return nd -} - -func mkdirP(t *testing.T, root *Directory, pth string) *Directory { - dirs := path.SplitList(pth) - cur := root - for _, d := range dirs { - n, err := cur.Mkdir(d) - if err != nil && err != os.ErrExist { - t.Fatal(err) - } - if err == os.ErrExist { - fsn, err := cur.Child(d) - if err != nil { - t.Fatal(err) - } - switch fsn := fsn.(type) { - case *Directory: - n = fsn - case *File: - t.Fatal("tried to make a directory where a file already exists") - } - } - - cur = n - } - return cur -} - -func assertDirAtPath(root *Directory, pth string, children []string) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fsn, err := DirLookup(root, pth) - if err != nil { - return err - } - - dir, ok := fsn.(*Directory) - if !ok { - return fmt.Errorf("%s was not a directory", pth) - } - - listing, err := dir.List(ctx) - if err != nil { - return err - } - - var names []string - for _, d := range listing { - names = append(names, d.Name) - } - - sort.Strings(children) - sort.Strings(names) - if !compStrArrs(children, names) { - return errors.New("directories children did not match") - } - - return nil -} - -func compStrArrs(a, b []string) bool { - if len(a) != len(b) { - return false - } - - for i := 0; i < len(a); i++ { - if a[i] != b[i] { - return false - } - } - - return true -} - -func assertFileAtPath(ds ipld.DAGService, root *Directory, expn ipld.Node, pth string) error { - exp, ok := expn.(*dag.ProtoNode) - if !ok { - return dag.ErrNotProtobuf - } - - parts := path.SplitList(pth) - cur := root - for i, d := range parts[:len(parts)-1] { - next, err := cur.Child(d) - if err != nil { - return fmt.Errorf("looking for %s failed: %s", pth, err) - } - - nextDir, ok := next.(*Directory) - if !ok { - return fmt.Errorf("%s points to a non-directory", parts[:i+1]) - } - - cur = nextDir - } - - last := parts[len(parts)-1] - finaln, err := cur.Child(last) - if err != nil { - return err - } - - file, ok := finaln.(*File) - if !ok { - return fmt.Errorf("%s was not a file", pth) - } - - rfd, err := file.Open(OpenReadOnly, false) - if err != nil { - return err - } - - out, err := ioutil.ReadAll(rfd) - if err != nil { - return err - } - - expbytes, err := catNode(ds, exp) - if err != nil { - return err - } - - if !bytes.Equal(out, expbytes) { - return fmt.Errorf("incorrect data at path") - } - return nil -} - -func catNode(ds ipld.DAGService, nd *dag.ProtoNode) ([]byte, error) { - r, err := uio.NewDagReader(context.TODO(), nd, ds) - if err != nil { - return nil, err - } - defer r.Close() - - return ioutil.ReadAll(r) -} - -func setupRoot(ctx context.Context, t *testing.T) (ipld.DAGService, *Root) { - ds := getDagserv(t) - - root := emptyDirNode() - rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, c *cid.Cid) error { - fmt.Println("PUBLISHED: ", c) - return nil - }) - - if err != nil { - t.Fatal(err) - } - - return ds, rt -} - -func TestBasic(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - // test making a basic dir - _, err := rootdir.Mkdir("a") - if err != nil { - t.Fatal(err) - } - - path := "a/b/c/d/e/f/g" - d := mkdirP(t, rootdir, path) - - fi := getRandFile(t, ds, 1000) - - // test inserting that file - err = d.AddChild("afile", fi) - if err != nil { - t.Fatal(err) - } - - err = assertFileAtPath(ds, rootdir, fi, "a/b/c/d/e/f/g/afile") - if err != nil { - t.Fatal(err) - } -} - -func TestMkdir(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - dirsToMake := []string{"a", "B", "foo", "bar", "cats", "fish"} - sort.Strings(dirsToMake) // sort for easy comparing later - - for _, d := range dirsToMake { - _, err := rootdir.Mkdir(d) - if err != nil { - t.Fatal(err) - } - } - - err := assertDirAtPath(rootdir, "/", dirsToMake) - if err != nil { - t.Fatal(err) - } - - for _, d := range dirsToMake { - mkdirP(t, rootdir, "a/"+d) - } - - err = assertDirAtPath(rootdir, "/a", dirsToMake) - if err != nil { - t.Fatal(err) - } - - // mkdir over existing dir should fail - _, err = rootdir.Mkdir("a") - if err == nil { - t.Fatal("should have failed!") - } -} - -func TestDirectoryLoadFromDag(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - nd := getRandFile(t, ds, 1000) - err := ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - - fihash := nd.Cid() - - dir := emptyDirNode() - err = ds.Add(ctx, dir) - if err != nil { - t.Fatal(err) - } - - dirhash := dir.Cid() - - top := emptyDirNode() - top.SetLinks([]*ipld.Link{ - { - Name: "a", - Cid: fihash, - }, - { - Name: "b", - Cid: dirhash, - }, - }) - - err = rootdir.AddChild("foo", top) - if err != nil { - t.Fatal(err) - } - - // get this dir - topi, err := rootdir.Child("foo") - if err != nil { - t.Fatal(err) - } - - topd := topi.(*Directory) - - path := topd.Path() - if path != "/foo" { - t.Fatalf("Expected path '/foo', got '%s'", path) - } - - // mkdir over existing but unloaded child file should fail - _, err = topd.Mkdir("a") - if err == nil { - t.Fatal("expected to fail!") - } - - // mkdir over existing but unloaded child dir should fail - _, err = topd.Mkdir("b") - if err == nil { - t.Fatal("expected to fail!") - } - - // adding a child over an existing path fails - err = topd.AddChild("b", nd) - if err == nil { - t.Fatal("expected to fail!") - } - - err = assertFileAtPath(ds, rootdir, nd, "foo/a") - if err != nil { - t.Fatal(err) - } - - err = assertDirAtPath(rootdir, "foo/b", nil) - if err != nil { - t.Fatal(err) - } - - err = rootdir.Unlink("foo") - if err != nil { - t.Fatal(err) - } - - err = assertDirAtPath(rootdir, "", nil) - if err != nil { - t.Fatal(err) - } -} - -func TestMfsFile(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - fisize := 1000 - nd := getRandFile(t, ds, 1000) - - err := rootdir.AddChild("file", nd) - if err != nil { - t.Fatal(err) - } - - fsn, err := rootdir.Child("file") - if err != nil { - t.Fatal(err) - } - - fi := fsn.(*File) - - if fi.Type() != TFile { - t.Fatal("some is seriously wrong here") - } - - wfd, err := fi.Open(OpenReadWrite, true) - if err != nil { - t.Fatal(err) - } - - // assert size is as expected - size, err := fi.Size() - if err != nil { - t.Fatal(err) - } - if size != int64(fisize) { - t.Fatal("size isnt correct") - } - - // write to beginning of file - b := []byte("THIS IS A TEST") - n, err := wfd.Write(b) - if err != nil { - t.Fatal(err) - } - - if n != len(b) { - t.Fatal("didnt write correct number of bytes") - } - - // sync file - err = wfd.Sync() - if err != nil { - t.Fatal(err) - } - - // make sure size hasnt changed - size, err = wfd.Size() - if err != nil { - t.Fatal(err) - } - if size != int64(fisize) { - t.Fatal("size isnt correct") - } - - // seek back to beginning - ns, err := wfd.Seek(0, io.SeekStart) - if err != nil { - t.Fatal(err) - } - - if ns != 0 { - t.Fatal("didnt seek to beginning") - } - - // read back bytes we wrote - buf := make([]byte, len(b)) - n, err = wfd.Read(buf) - if err != nil { - t.Fatal(err) - } - - if n != len(buf) { - t.Fatal("didnt read enough") - } - - if !bytes.Equal(buf, b) { - t.Fatal("data read was different than data written") - } - - // truncate file to ten bytes - err = wfd.Truncate(10) - if err != nil { - t.Fatal(err) - } - - size, err = wfd.Size() - if err != nil { - t.Fatal(err) - } - - if size != 10 { - t.Fatal("size was incorrect: ", size) - } - - // 'writeAt' to extend it - data := []byte("this is a test foo foo foo") - nwa, err := wfd.WriteAt(data, 5) - if err != nil { - t.Fatal(err) - } - - if nwa != len(data) { - t.Fatal(err) - } - - // assert size once more - size, err = wfd.Size() - if err != nil { - t.Fatal(err) - } - - if size != int64(5+len(data)) { - t.Fatal("size was incorrect") - } - - // close it out! - err = wfd.Close() - if err != nil { - t.Fatal(err) - } - - // make sure we can get node. TODO: verify it later - _, err = fi.GetNode() - if err != nil { - t.Fatal(err) - } -} - -func randomWalk(d *Directory, n int) (*Directory, error) { - for i := 0; i < n; i++ { - dirents, err := d.List(context.Background()) - if err != nil { - return nil, err - } - - var childdirs []NodeListing - for _, child := range dirents { - if child.Type == int(TDir) { - childdirs = append(childdirs, child) - } - } - if len(childdirs) == 0 { - return d, nil - } - - next := childdirs[rand.Intn(len(childdirs))].Name - - nextD, err := d.Child(next) - if err != nil { - return nil, err - } - - d = nextD.(*Directory) - } - return d, nil -} - -func randomName() string { - set := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_" - length := rand.Intn(10) + 2 - var out string - for i := 0; i < length; i++ { - j := rand.Intn(len(set)) - out += set[j : j+1] - } - return out -} - -func actorMakeFile(d *Directory) error { - d, err := randomWalk(d, rand.Intn(7)) - if err != nil { - return err - } - - name := randomName() - f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dserv) - if err != nil { - return err - } - - wfd, err := f.Open(OpenWriteOnly, true) - if err != nil { - return err - } - - rread := rand.New(rand.NewSource(time.Now().UnixNano())) - r := io.LimitReader(rread, int64(77*rand.Intn(123))) - _, err = io.Copy(wfd, r) - if err != nil { - return err - } - - return wfd.Close() -} - -func actorMkdir(d *Directory) error { - d, err := randomWalk(d, rand.Intn(7)) - if err != nil { - return err - } - - _, err = d.Mkdir(randomName()) - - return err -} - -func randomFile(d *Directory) (*File, error) { - d, err := randomWalk(d, rand.Intn(6)) - if err != nil { - return nil, err - } - - ents, err := d.List(context.Background()) - if err != nil { - return nil, err - } - - var files []string - for _, e := range ents { - if e.Type == int(TFile) { - files = append(files, e.Name) - } - } - - if len(files) == 0 { - return nil, nil - } - - fname := files[rand.Intn(len(files))] - fsn, err := d.Child(fname) - if err != nil { - return nil, err - } - - fi, ok := fsn.(*File) - if !ok { - return nil, errors.New("file wasn't a file, race?") - } - - return fi, nil -} - -func actorWriteFile(d *Directory) error { - fi, err := randomFile(d) - if err != nil { - return err - } - if fi == nil { - return nil - } - - size := rand.Intn(1024) + 1 - buf := make([]byte, size) - rand.Read(buf) - - s, err := fi.Size() - if err != nil { - return err - } - - wfd, err := fi.Open(OpenWriteOnly, true) - if err != nil { - return err - } - - offset := rand.Int63n(s) - - n, err := wfd.WriteAt(buf, offset) - if err != nil { - return err - } - if n != size { - return fmt.Errorf("didnt write enough") - } - - return wfd.Close() -} - -func actorReadFile(d *Directory) error { - fi, err := randomFile(d) - if err != nil { - return err - } - if fi == nil { - return nil - } - - _, err = fi.Size() - if err != nil { - return err - } - - rfd, err := fi.Open(OpenReadOnly, false) - if err != nil { - return err - } - - _, err = ioutil.ReadAll(rfd) - if err != nil { - return err - } - - return rfd.Close() -} - -func testActor(rt *Root, iterations int, errs chan error) { - d := rt.GetDirectory() - for i := 0; i < iterations; i++ { - switch rand.Intn(5) { - case 0: - if err := actorMkdir(d); err != nil { - errs <- err - return - } - case 1, 2: - if err := actorMakeFile(d); err != nil { - errs <- err - return - } - case 3: - if err := actorWriteFile(d); err != nil { - errs <- err - return - } - case 4: - if err := actorReadFile(d); err != nil { - errs <- err - return - } - } - } - errs <- nil -} - -func TestMfsStress(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, rt := setupRoot(ctx, t) - - numroutines := 10 - - errs := make(chan error) - for i := 0; i < numroutines; i++ { - go testActor(rt, 50, errs) - } - - for i := 0; i < numroutines; i++ { - err := <-errs - if err != nil { - t.Fatal(err) - } - } -} - -func TestMfsHugeDir(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, rt := setupRoot(ctx, t) - - for i := 0; i < 10000; i++ { - err := Mkdir(rt, fmt.Sprintf("/dir%d", i), MkdirOpts{Mkparents: false, Flush: false}) - if err != nil { - t.Fatal(err) - } - } -} - -func TestMkdirP(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, rt := setupRoot(ctx, t) - - err := Mkdir(rt, "/a/b/c/d/e/f", MkdirOpts{Mkparents: true, Flush: true}) - if err != nil { - t.Fatal(err) - } -} - -func TestConcurrentWriteAndFlush(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - d := mkdirP(t, rt.GetDirectory(), "foo/bar/baz") - fn := fileNodeFromReader(t, ds, bytes.NewBuffer(nil)) - err := d.AddChild("file", fn) - if err != nil { - t.Fatal(err) - } - - nloops := 5000 - - wg := new(sync.WaitGroup) - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < nloops; i++ { - err := writeFile(rt, "/foo/bar/baz/file", []byte("STUFF")) - if err != nil { - t.Error("file write failed: ", err) - return - } - } - }() - - for i := 0; i < nloops; i++ { - _, err := rt.GetDirectory().GetNode() - if err != nil { - t.Fatal(err) - } - } - - wg.Wait() -} - -func TestFlushing(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, rt := setupRoot(ctx, t) - - dir := rt.GetDirectory() - c := mkdirP(t, dir, "a/b/c") - d := mkdirP(t, dir, "a/b/d") - e := mkdirP(t, dir, "a/b/e") - - data := []byte("this is a test\n") - nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data)))) - - if err := c.AddChild("TEST", nd1); err != nil { - t.Fatal(err) - } - if err := d.AddChild("TEST", nd1); err != nil { - t.Fatal(err) - } - if err := e.AddChild("TEST", nd1); err != nil { - t.Fatal(err) - } - if err := dir.AddChild("FILE", nd1); err != nil { - t.Fatal(err) - } - - if err := FlushPath(rt, "/a/b/c/TEST"); err != nil { - t.Fatal(err) - } - - if err := FlushPath(rt, "/a/b/d/TEST"); err != nil { - t.Fatal(err) - } - - if err := FlushPath(rt, "/a/b/e/TEST"); err != nil { - t.Fatal(err) - } - - if err := FlushPath(rt, "/FILE"); err != nil { - t.Fatal(err) - } - - rnd, err := dir.GetNode() - if err != nil { - t.Fatal(err) - } - - pbrnd, ok := rnd.(*dag.ProtoNode) - if !ok { - t.Fatal(dag.ErrNotProtobuf) - } - - fsnode, err := ft.FSNodeFromBytes(pbrnd.Data()) - if err != nil { - t.Fatal(err) - } - - if fsnode.Type() != ft.TDirectory { - t.Fatal("root wasnt a directory") - } - - rnk := rnd.Cid() - exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr" - if rnk.String() != exp { - t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String()) - } -} - -func readFile(rt *Root, path string, offset int64, buf []byte) error { - n, err := Lookup(rt, path) - if err != nil { - return err - } - - fi, ok := n.(*File) - if !ok { - return fmt.Errorf("%s was not a file", path) - } - - fd, err := fi.Open(OpenReadOnly, false) - if err != nil { - return err - } - - _, err = fd.Seek(offset, io.SeekStart) - if err != nil { - return err - } - - nread, err := fd.Read(buf) - if err != nil { - return err - } - if nread != len(buf) { - return fmt.Errorf("didn't read enough") - } - - return fd.Close() -} - -func TestConcurrentReads(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ds, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - path := "a/b/c" - d := mkdirP(t, rootdir, path) - - buf := make([]byte, 2048) - rand.Read(buf) - - fi := fileNodeFromReader(t, ds, bytes.NewReader(buf)) - err := d.AddChild("afile", fi) - if err != nil { - t.Fatal(err) - } - - var wg sync.WaitGroup - nloops := 100 - for i := 0; i < 10; i++ { - wg.Add(1) - go func(me int) { - defer wg.Done() - mybuf := make([]byte, len(buf)) - for j := 0; j < nloops; j++ { - offset := rand.Intn(len(buf)) - length := rand.Intn(len(buf) - offset) - - err := readFile(rt, "/a/b/c/afile", int64(offset), mybuf[:length]) - if err != nil { - t.Error("readfile failed: ", err) - return - } - - if !bytes.Equal(mybuf[:length], buf[offset:offset+length]) { - t.Error("incorrect read!") - } - } - }(i) - } - wg.Wait() -} - -func writeFile(rt *Root, path string, data []byte) error { - n, err := Lookup(rt, path) - if err != nil { - return err - } - - fi, ok := n.(*File) - if !ok { - return fmt.Errorf("expected to receive a file, but didnt get one") - } - - fd, err := fi.Open(OpenWriteOnly, true) - if err != nil { - return err - } - defer fd.Close() - - nw, err := fd.Write(data) - if err != nil { - return err - } - - if nw != len(data) { - return fmt.Errorf("wrote incorrect amount: %d != 10", nw) - } - - return nil -} - -func TestConcurrentWrites(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ds, rt := setupRoot(ctx, t) - - rootdir := rt.GetDirectory() - - path := "a/b/c" - d := mkdirP(t, rootdir, path) - - fi := fileNodeFromReader(t, ds, bytes.NewReader(make([]byte, 0))) - err := d.AddChild("afile", fi) - if err != nil { - t.Fatal(err) - } - - var wg sync.WaitGroup - nloops := 100 - for i := 0; i < 10; i++ { - wg.Add(1) - go func(me int) { - defer wg.Done() - mybuf := bytes.Repeat([]byte{byte(me)}, 10) - for j := 0; j < nloops; j++ { - err := writeFile(rt, "a/b/c/afile", mybuf) - if err != nil { - t.Error("writefile failed: ", err) - return - } - } - }(i) - } - wg.Wait() -} - -func TestFileDescriptors(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ds, rt := setupRoot(ctx, t) - dir := rt.GetDirectory() - - nd := dag.NodeWithData(ft.FilePBData(nil, 0)) - fi, err := NewFile("test", nd, dir, ds) - if err != nil { - t.Fatal(err) - } - - // test read only - rfd1, err := fi.Open(OpenReadOnly, false) - if err != nil { - t.Fatal(err) - } - - err = rfd1.Truncate(0) - if err == nil { - t.Fatal("shouldnt be able to truncate readonly fd") - } - - _, err = rfd1.Write([]byte{}) - if err == nil { - t.Fatal("shouldnt be able to write to readonly fd") - } - - _, err = rfd1.Read([]byte{}) - if err != nil { - t.Fatalf("expected to be able to read from file: %s", err) - } - - done := make(chan struct{}) - go func() { - defer close(done) - // can open second readonly file descriptor - rfd2, err := fi.Open(OpenReadOnly, false) - if err != nil { - t.Error(err) - return - } - - rfd2.Close() - }() - - select { - case <-time.After(time.Second): - t.Fatal("open second file descriptor failed") - case <-done: - } - - if t.Failed() { - return - } - - // test not being able to open for write until reader are closed - done = make(chan struct{}) - go func() { - defer close(done) - wfd1, err := fi.Open(OpenWriteOnly, true) - if err != nil { - t.Error(err) - } - - wfd1.Close() - }() - - select { - case <-time.After(time.Millisecond * 200): - case <-done: - if t.Failed() { - return - } - - t.Fatal("shouldnt have been able to open file for writing") - } - - err = rfd1.Close() - if err != nil { - t.Fatal(err) - } - - select { - case <-time.After(time.Second): - t.Fatal("should have been able to open write fd after closing read fd") - case <-done: - } - - wfd, err := fi.Open(OpenWriteOnly, true) - if err != nil { - t.Fatal(err) - } - - _, err = wfd.Read([]byte{}) - if err == nil { - t.Fatal("shouldnt have been able to read from write only filedescriptor") - } - - _, err = wfd.Write([]byte{}) - if err != nil { - t.Fatal(err) - } -} - -func TestTruncateAtSize(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - dir := rt.GetDirectory() - - nd := dag.NodeWithData(ft.FilePBData(nil, 0)) - fi, err := NewFile("test", nd, dir, ds) - if err != nil { - t.Fatal(err) - } - - fd, err := fi.Open(OpenReadWrite, true) - if err != nil { - t.Fatal(err) - } - defer fd.Close() - _, err = fd.Write([]byte("test")) - if err != nil { - t.Fatal(err) - } - fd.Truncate(4) -} - -func TestTruncateAndWrite(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ds, rt := setupRoot(ctx, t) - - dir := rt.GetDirectory() - - nd := dag.NodeWithData(ft.FilePBData(nil, 0)) - fi, err := NewFile("test", nd, dir, ds) - if err != nil { - t.Fatal(err) - } - - fd, err := fi.Open(OpenReadWrite, true) - defer fd.Close() - if err != nil { - t.Fatal(err) - } - for i := 0; i < 200; i++ { - err = fd.Truncate(0) - if err != nil { - t.Fatal(err) - } - l, err := fd.Write([]byte("test")) - if err != nil { - t.Fatal(err) - } - if l != len("test") { - t.Fatal("incorrect write length") - } - - _, err = fd.Seek(0, io.SeekStart) - if err != nil { - t.Fatal(err) - } - - data, err := ioutil.ReadAll(fd) - if err != nil { - t.Fatal(err) - } - if string(data) != "test" { - t.Fatalf("read error at read %d, read: %v", i, data) - } - } -} diff --git a/mfs/ops.go b/mfs/ops.go deleted file mode 100644 index c90071fb8..000000000 --- a/mfs/ops.go +++ /dev/null @@ -1,223 +0,0 @@ -package mfs - -import ( - "fmt" - "os" - gopath "path" - "strings" - - path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path" - - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format" -) - -// Mv moves the file or directory at 'src' to 'dst' -func Mv(r *Root, src, dst string) error { - srcDir, srcFname := gopath.Split(src) - - var dstDirStr string - var filename string - if dst[len(dst)-1] == '/' { - dstDirStr = dst - filename = srcFname - } else { - dstDirStr, filename = gopath.Split(dst) - } - - // get parent directories of both src and dest first - dstDir, err := lookupDir(r, dstDirStr) - if err != nil { - return err - } - - srcDirObj, err := lookupDir(r, srcDir) - if err != nil { - return err - } - - srcObj, err := srcDirObj.Child(srcFname) - if err != nil { - return err - } - - nd, err := srcObj.GetNode() - if err != nil { - return err - } - - fsn, err := dstDir.Child(filename) - if err == nil { - switch n := fsn.(type) { - case *File: - _ = dstDir.Unlink(filename) - case *Directory: - dstDir = n - default: - return fmt.Errorf("unexpected type at path: %s", dst) - } - } else if err != os.ErrNotExist { - return err - } - - err = dstDir.AddChild(filename, nd) - if err != nil { - return err - } - - return srcDirObj.Unlink(srcFname) -} - -func lookupDir(r *Root, path string) (*Directory, error) { - di, err := Lookup(r, path) - if err != nil { - return nil, err - } - - d, ok := di.(*Directory) - if !ok { - return nil, fmt.Errorf("%s is not a directory", path) - } - - return d, nil -} - -// PutNode inserts 'nd' at 'path' in the given mfs -func PutNode(r *Root, path string, nd ipld.Node) error { - dirp, filename := gopath.Split(path) - if filename == "" { - return fmt.Errorf("cannot create file with empty name") - } - - pdir, err := lookupDir(r, dirp) - if err != nil { - return err - } - - return pdir.AddChild(filename, nd) -} - -// MkdirOpts is used by Mkdir -type MkdirOpts struct { - Mkparents bool - Flush bool - CidBuilder cid.Builder -} - -// Mkdir creates a directory at 'path' under the directory 'd', creating -// intermediary directories as needed if 'mkparents' is set to true -func Mkdir(r *Root, pth string, opts MkdirOpts) error { - if pth == "" { - return fmt.Errorf("no path given to Mkdir") - } - parts := path.SplitList(pth) - if parts[0] == "" { - parts = parts[1:] - } - - // allow 'mkdir /a/b/c/' to create c - if parts[len(parts)-1] == "" { - parts = parts[:len(parts)-1] - } - - if len(parts) == 0 { - // this will only happen on 'mkdir /' - if opts.Mkparents { - return nil - } - return fmt.Errorf("cannot create directory '/': Already exists") - } - - cur := r.GetDirectory() - for i, d := range parts[:len(parts)-1] { - fsn, err := cur.Child(d) - if err == os.ErrNotExist && opts.Mkparents { - mkd, err := cur.Mkdir(d) - if err != nil { - return err - } - if opts.CidBuilder != nil { - mkd.SetCidBuilder(opts.CidBuilder) - } - fsn = mkd - } else if err != nil { - return err - } - - next, ok := fsn.(*Directory) - if !ok { - return fmt.Errorf("%s was not a directory", path.Join(parts[:i])) - } - cur = next - } - - final, err := cur.Mkdir(parts[len(parts)-1]) - if err != nil { - if !opts.Mkparents || err != os.ErrExist || final == nil { - return err - } - } - if opts.CidBuilder != nil { - final.SetCidBuilder(opts.CidBuilder) - } - - if opts.Flush { - err := final.Flush() - if err != nil { - return err - } - } - - return nil -} - -// Lookup extracts the root directory and performs a lookup under it. -// TODO: Now that the root is always a directory, can this function -// be collapsed with `DirLookup`? Or at least be made a method of `Root`? -func Lookup(r *Root, path string) (FSNode, error) { - dir := r.GetDirectory() - - return DirLookup(dir, path) -} - -// DirLookup will look up a file or directory at the given path -// under the directory 'd' -func DirLookup(d *Directory, pth string) (FSNode, error) { - pth = strings.Trim(pth, "/") - parts := path.SplitList(pth) - if len(parts) == 1 && parts[0] == "" { - return d, nil - } - - var cur FSNode - cur = d - for i, p := range parts { - chdir, ok := cur.(*Directory) - if !ok { - return nil, fmt.Errorf("cannot access %s: Not a directory", path.Join(parts[:i+1])) - } - - child, err := chdir.Child(p) - if err != nil { - return nil, err - } - - cur = child - } - return cur, nil -} - -func FlushPath(rt *Root, pth string) error { - nd, err := Lookup(rt, pth) - if err != nil { - return err - } - - err = nd.Flush() - if err != nil { - return err - } - - rt.repub.WaitPub() - return nil -} diff --git a/mfs/repub_test.go b/mfs/repub_test.go deleted file mode 100644 index 123c20859..000000000 --- a/mfs/repub_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package mfs - -import ( - "context" - "testing" - "time" - - ci "gx/ipfs/QmXG74iiKQnDstVQq9fPFQEB6JTNSWBbAWE1qsq6L4E5sR/go-testutil/ci" - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" -) - -func TestRepublisher(t *testing.T) { - if ci.IsRunning() { - t.Skip("dont run timing tests in CI") - } - - ctx := context.TODO() - - pub := make(chan struct{}) - - pf := func(ctx context.Context, c *cid.Cid) error { - pub <- struct{}{} - return nil - } - - tshort := time.Millisecond * 50 - tlong := time.Second / 2 - - rp := NewRepublisher(ctx, pf, tshort, tlong) - go rp.Run() - - rp.Update(nil) - - // should hit short timeout - select { - case <-time.After(tshort * 2): - t.Fatal("publish didnt happen in time") - case <-pub: - } - - cctx, cancel := context.WithCancel(context.Background()) - - go func() { - for { - rp.Update(nil) - time.Sleep(time.Millisecond * 10) - select { - case <-cctx.Done(): - return - default: - } - } - }() - - select { - case <-pub: - t.Fatal("shouldnt have received publish yet!") - case <-time.After((tlong * 9) / 10): - } - select { - case <-pub: - case <-time.After(tlong / 2): - t.Fatal("waited too long for pub!") - } - - cancel() - - go func() { - err := rp.Close() - if err != nil { - t.Fatal(err) - } - }() - - // final pub from closing - <-pub -} diff --git a/mfs/system.go b/mfs/system.go deleted file mode 100644 index 2e7c400c9..000000000 --- a/mfs/system.go +++ /dev/null @@ -1,292 +0,0 @@ -// package mfs implements an in memory model of a mutable IPFS filesystem. -// -// It consists of four main structs: -// 1) The Filesystem -// The filesystem serves as a container and entry point for various mfs filesystems -// 2) Root -// Root represents an individual filesystem mounted within the mfs system as a whole -// 3) Directories -// 4) Files -package mfs - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag" - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs" - - logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid" - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format" -) - -var ErrNotExist = errors.New("no such rootfs") - -var log = logging.Logger("mfs") - -var ErrIsDirectory = errors.New("error: is a directory") - -type childCloser interface { - closeChild(string, ipld.Node, bool) error -} - -type NodeType int - -const ( - TFile NodeType = iota - TDir -) - -// FSNode represents any node (directory, root, or file) in the mfs filesystem. -type FSNode interface { - GetNode() (ipld.Node, error) - Flush() error - Type() NodeType -} - -// Root represents the root of a filesystem tree. -type Root struct { - - // Root directory of the MFS layout. - dir *Directory - - repub *Republisher -} - -// PubFunc is the function used by the `publish()` method. -type PubFunc func(context.Context, *cid.Cid) error - -// NewRoot creates a new Root and starts up a republisher routine for it. -func NewRoot(parent context.Context, ds ipld.DAGService, node *dag.ProtoNode, pf PubFunc) (*Root, error) { - - var repub *Republisher - if pf != nil { - repub = NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3) - repub.setVal(node.Cid()) - go repub.Run() - } - - root := &Root{ - repub: repub, - } - - pbn, err := ft.FromBytes(node.Data()) - if err != nil { - log.Error("IPNS pointer was not unixfs node") - return nil, err - } - - switch pbn.GetType() { - case ft.TDirectory, ft.THAMTShard: - newDir, err := NewDirectory(parent, node.String(), node, root, ds) - if err != nil { - return nil, err - } - - root.dir = newDir - case ft.TFile, ft.TMetadata, ft.TRaw: - return nil, fmt.Errorf("root can't be a file (unixfs type: %s)", pbn.GetType()) - default: - return nil, fmt.Errorf("unrecognized unixfs type: %s", pbn.GetType()) - } - return root, nil -} - -// GetDirectory returns the root directory. -func (kr *Root) GetDirectory() *Directory { - return kr.dir -} - -// Flush signals that an update has occurred since the last publish, -// and updates the Root republisher. -func (kr *Root) Flush() error { - nd, err := kr.GetDirectory().GetNode() - if err != nil { - return err - } - - if kr.repub != nil { - kr.repub.Update(nd.Cid()) - } - return nil -} - -// FlushMemFree flushes the root directory and then uncaches all of its links. -// This has the effect of clearing out potentially stale references and allows -// them to be garbage collected. -// CAUTION: Take care not to ever call this while holding a reference to any -// child directories. Those directories will be bad references and using them -// may have unintended racy side effects. -// A better implemented mfs system (one that does smarter internal caching and -// refcounting) shouldnt need this method. -func (kr *Root) FlushMemFree(ctx context.Context) error { - dir := kr.GetDirectory() - - if err := dir.Flush(); err != nil { - return err - } - - dir.lock.Lock() - defer dir.lock.Unlock() - for name := range dir.files { - delete(dir.files, name) - } - for name := range dir.childDirs { - delete(dir.childDirs, name) - } - - return nil -} - -// closeChild implements the childCloser interface, and signals to the publisher that -// there are changes ready to be published. -func (kr *Root) closeChild(name string, nd ipld.Node, sync bool) error { - err := kr.GetDirectory().dserv.Add(context.TODO(), nd) - if err != nil { - return err - } - - if kr.repub != nil { - kr.repub.Update(nd.Cid()) - } - return nil -} - -func (kr *Root) Close() error { - nd, err := kr.GetDirectory().GetNode() - if err != nil { - return err - } - - if kr.repub != nil { - kr.repub.Update(nd.Cid()) - return kr.repub.Close() - } - - return nil -} - -// Republisher manages when to publish a given entry. -type Republisher struct { - TimeoutLong time.Duration - TimeoutShort time.Duration - Publish chan struct{} - pubfunc PubFunc - pubnowch chan chan struct{} - - ctx context.Context - cancel func() - - lk sync.Mutex - val *cid.Cid - lastpub *cid.Cid -} - -// NewRepublisher creates a new Republisher object to republish the given root -// using the given short and long time intervals. -func NewRepublisher(ctx context.Context, pf PubFunc, tshort, tlong time.Duration) *Republisher { - ctx, cancel := context.WithCancel(ctx) - return &Republisher{ - TimeoutShort: tshort, - TimeoutLong: tlong, - Publish: make(chan struct{}, 1), - pubfunc: pf, - pubnowch: make(chan chan struct{}), - ctx: ctx, - cancel: cancel, - } -} - -func (p *Republisher) setVal(c *cid.Cid) { - p.lk.Lock() - defer p.lk.Unlock() - p.val = c -} - -// WaitPub Returns immediately if `lastpub` value is consistent with the -// current value `val`, else will block until `val` has been published. -func (p *Republisher) WaitPub() { - p.lk.Lock() - consistent := p.lastpub == p.val - p.lk.Unlock() - if consistent { - return - } - - wait := make(chan struct{}) - p.pubnowch <- wait - <-wait -} - -func (p *Republisher) Close() error { - err := p.publish(p.ctx) - p.cancel() - return err -} - -// Touch signals that an update has occurred since the last publish. -// Multiple consecutive touches may extend the time period before -// the next Publish occurs in order to more efficiently batch updates. -func (np *Republisher) Update(c *cid.Cid) { - np.setVal(c) - select { - case np.Publish <- struct{}{}: - default: - } -} - -// Run is the main republisher loop. -func (np *Republisher) Run() { - for { - select { - case <-np.Publish: - quick := time.After(np.TimeoutShort) - longer := time.After(np.TimeoutLong) - - wait: - var pubnowresp chan struct{} - - select { - case <-np.ctx.Done(): - return - case <-np.Publish: - quick = time.After(np.TimeoutShort) - goto wait - case <-quick: - case <-longer: - case pubnowresp = <-np.pubnowch: - } - - err := np.publish(np.ctx) - if pubnowresp != nil { - pubnowresp <- struct{}{} - } - if err != nil { - log.Errorf("republishRoot error: %s", err) - } - - case <-np.ctx.Done(): - return - } - } -} - -// publish calls the `PubFunc`. -func (np *Republisher) publish(ctx context.Context) error { - np.lk.Lock() - topub := np.val - np.lk.Unlock() - - err := np.pubfunc(ctx, topub) - if err != nil { - return err - } - np.lk.Lock() - np.lastpub = topub - np.lk.Unlock() - return nil -} diff --git a/package.json b/package.json index d96d52672..183441b5d 100644 --- a/package.json +++ b/package.json @@ -527,6 +527,12 @@ "hash": "QmfMirfpEKQFctVpBYTvETxxLoU5q4ZJWsAMrtwSSE2bkn", "name": "go-verifcid", "version": "0.0.3" + }, + { + "author": "hsanjuan", + "hash": "QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc", + "name": "go-mfs", + "version": "0.0.1" } ], "gxVersion": "0.10.0", @@ -535,3 +541,4 @@ "name": "go-ipfs", "version": "0.4.18-dev" } +