From 9215955ee81aeb528de5109854646c202d47d4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 13 Mar 2019 16:23:58 +0100 Subject: [PATCH 1/3] coreapi: initial MFS impl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/coreapi.go | 7 ++ core/coreapi/mfs.go | 170 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 core/coreapi/mfs.go diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index c5ba1b566..f6c0fe5a6 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -17,6 +17,7 @@ import ( "context" "errors" "fmt" + "github.com/ipfs/go-mfs" "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/namesys" @@ -71,6 +72,8 @@ type CoreAPI struct { checkPublishAllowed func() error checkOnline func(allowOffline bool) error + filesRoot *mfs.Root // TODO: option filtering + // ONLY for re-applying options in WithOptions, DO NOT USE ANYWHERE ELSE nd *core.IpfsNode parentOpts options.ApiSettings @@ -119,6 +122,10 @@ func (api *CoreAPI) Object() coreiface.ObjectAPI { return (*ObjectAPI)(api) } +func (api *CoreAPI) Mfs() coreiface.MfsAPI { + return (*MfsAPI)(api) +} + // Pin returns the PinAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Pin() coreiface.PinAPI { return (*PinAPI)(api) diff --git a/core/coreapi/mfs.go b/core/coreapi/mfs.go new file mode 100644 index 000000000..9a4137df4 --- /dev/null +++ b/core/coreapi/mfs.go @@ -0,0 +1,170 @@ +package coreapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-mfs" + "github.com/pkg/errors" + "os" + gopath "path" + "time" + + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +const defaultFilePerm = 0644 +const defaultDirPerm = 0755 + +type MfsAPI CoreAPI + +type fileNodeInfo struct{ + l mfs.NodeListing +} + +func (i *fileNodeInfo) Name() string { + return i.l.Name //TODO: check what is really returned here +} + +func (i *fileNodeInfo) Size() int64 { + return i.l.Size +} + +func (i *fileNodeInfo) Mode() os.FileMode { + return defaultFilePerm +} + +func (i *fileNodeInfo) ModTime() time.Time { + return time.Unix(0, 0) +} + +func (i *fileNodeInfo) IsDir() bool { + return mfs.NodeType(i.l.Type) == mfs.TDir +} + +func (i *fileNodeInfo) Sys() interface{} { + return i.l +} + +func (api *MfsAPI) Create(ctx context.Context, path coreiface.MfsPath) (coreiface.File, error) { + return api.OpenFile(ctx, path, os.O_CREATE | os.O_WRONLY, defaultFilePerm) +} + +func (api *MfsAPI) Open(ctx context.Context, path coreiface.MfsPath) (coreiface.File, error) { + return api.OpenFile(ctx, path, os.O_RDWR, defaultFilePerm) +} + +func (api *MfsAPI) OpenFile(ctx context.Context, path coreiface.MfsPath, flag int, perm os.FileMode) (coreiface.File, error) { + panic("implement me") +} + +func (api *MfsAPI) Stat(ctx context.Context, path coreiface.MfsPath) (os.FileInfo, error) { + fsn, err := mfs.Lookup(api.filesRoot, path.String()) + if err != nil { + return nil, err + } + + nd, err := fsn.GetNode() + if err != nil { + return nil, err + } + + return &fileNodeInfo{ + l: mfs.NodeListing{ + Name: path.String(), + Type: int(fsn.Type()), + Size: -1, //TODO + Hash: nd.Cid().String(), + }, + }, nil +} + +func (api *MfsAPI) Rename(ctx context.Context, oldpath, newpath coreiface.MfsPath) error { + flush := false //TODO + err := mfs.Mv(api.filesRoot, oldpath.String(), newpath.String()) + if err == nil && flush { + err = mfs.FlushPath(api.filesRoot, "/") + } + return err +} + +func (api *MfsAPI) Remove(ctx context.Context, path coreiface.MfsPath) error { + if path.String() == "/" { + return fmt.Errorf("cannot delete root") + } + + dir, name := gopath.Split(path.String()) + parent, err := mfs.Lookup(api.filesRoot, dir) + if err != nil { + return fmt.Errorf("parent lookup: %s", err) + } + + pdir, ok := parent.(*mfs.Directory) + if !ok { + return fmt.Errorf("no such file or directory: %s", path) + } + + // TODO: force + + // get child node by name, when the node is corrupted and nonexistent, + // it will return specific error. + child, err := pdir.Child(name) + if err != nil { + return err + } + + dashr := true // TODO: make into an option + + switch child.(type) { + case *mfs.Directory: + if !dashr { + return fmt.Errorf("%s is a directory, use -r to remove directories", path) + } + } + + err = pdir.Unlink(name) + if err != nil { + return err + } + + return pdir.Flush() //TODO: setting for flush +} + +func (api *MfsAPI) ReadDir(ctx context.Context, path coreiface.MfsPath) ([]os.FileInfo, error) { + fsn, err := mfs.Lookup(api.filesRoot, path.String()) + if err != nil { + return nil, err + } + + switch fsn := fsn.(type) { + case *mfs.Directory: + lst, err := fsn.List(ctx) + if err != nil { + return nil, err + } + + out := make([]os.FileInfo, len(lst)) + for i, v := range lst { + out[i] = &fileNodeInfo{ + l: v, + } + } + + return out, nil + default: + return nil, errors.New("readdir: unsupported node type") + } +} + +func (api *MfsAPI) MkdirAll(path coreiface.MfsPath, perm os.FileMode) error { + return mfs.Mkdir(api.filesRoot, path.String(), mfs.MkdirOpts{ + Mkparents: true, + Flush: false, + //CidBuilder: prefix, TODO + }) +} + +func (api *MfsAPI) core() coreiface.CoreAPI { + return (*CoreAPI)(api) +} + +var _ os.FileInfo = &fileNodeInfo{} From 65a7dbaf97953be021d3b1fdfb3743eab889b154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Mar 2019 15:23:24 +0100 Subject: [PATCH 2/3] coreapi mfs: initial file implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/mfs.go | 97 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/core/coreapi/mfs.go b/core/coreapi/mfs.go index 9a4137df4..fa901ac73 100644 --- a/core/coreapi/mfs.go +++ b/core/coreapi/mfs.go @@ -5,8 +5,10 @@ import ( "fmt" "github.com/ipfs/go-mfs" "github.com/pkg/errors" + "io" "os" gopath "path" + "sync" "time" coreiface "github.com/ipfs/interface-go-ipfs-core" @@ -53,10 +55,101 @@ func (api *MfsAPI) Open(ctx context.Context, path coreiface.MfsPath) (coreiface. return api.OpenFile(ctx, path, os.O_RDWR, defaultFilePerm) } -func (api *MfsAPI) OpenFile(ctx context.Context, path coreiface.MfsPath, flag int, perm os.FileMode) (coreiface.File, error) { +type mfsFile struct { + mfs.FileDescriptor + lk sync.Mutex // Only needed for ReadAt, remove when it's in go-mfs +} + +func (f *mfsFile) Read(p []byte) (n int, err error) { + f.lk.Lock() + defer f.lk.Unlock() + return f.FileDescriptor.Read(p) +} + +func (f *mfsFile) Write(p []byte) (n int, err error) { + f.lk.Lock() + defer f.lk.Unlock() + return f.FileDescriptor.Write(p) +} + +func (f *mfsFile) Seek(offset int64, whence int) (int64, error) { + f.lk.Lock() + defer f.lk.Unlock() + return f.FileDescriptor.Seek(offset, whence) +} + +func (f *mfsFile) Close() error { + f.lk.Lock() + defer f.lk.Unlock() + return f.FileDescriptor.Close() +} + + +func (f *mfsFile) ReadAt(p []byte, off int64) (int, error) { + // TODO: implement in MFS with less locking + f.lk.Lock() + defer f.lk.Unlock() + + cur, err := f.FileDescriptor.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + + so, err := f.FileDescriptor.Seek(off, io.SeekStart) + if err != nil { + return 0, err + } + + if so != off { + return 0, errors.New("seek to wrong offset") + } + + n, err := f.FileDescriptor.Read(p) + if err != nil { + return n, err + } + + off, err = f.FileDescriptor.Seek(cur, io.SeekStart) + if err != nil { + return n, err + } + + if cur != off { + return n, errors.New("seek to wrong offset") + } + + return n, nil +} + +func (f *mfsFile) Name() coreiface.MfsPath { panic("implement me") } +func (api *MfsAPI) OpenFile(ctx context.Context, path coreiface.MfsPath, flag int, perm os.FileMode) (coreiface.File, error) { + fsn, err := mfs.Lookup(api.filesRoot, path.String()) + if err != nil { + return nil, err + } + + fn, ok := fsn.(*mfs.File) + if !ok { + return nil, fmt.Errorf("cannot open %s: not a file", path.String()) + } + + flags := mfs.Flags{ + Read: flag & os.O_WRONLY == 0, + Write: flag & (os.O_WRONLY | os.O_RDWR) != 0, + Sync: flag & os.O_SYNC != 0, + } + + _, err = fn.Open(flags) + if err != nil { + return nil, err + } + + return &mfsFile{}, nil +} + func (api *MfsAPI) Stat(ctx context.Context, path coreiface.MfsPath) (os.FileInfo, error) { fsn, err := mfs.Lookup(api.filesRoot, path.String()) if err != nil { @@ -155,7 +248,7 @@ func (api *MfsAPI) ReadDir(ctx context.Context, path coreiface.MfsPath) ([]os.Fi } } -func (api *MfsAPI) MkdirAll(path coreiface.MfsPath, perm os.FileMode) error { +func (api *MfsAPI) MkdirAll(ctx context.Context, path coreiface.MfsPath, perm os.FileMode) error { return mfs.Mkdir(api.filesRoot, path.String(), mfs.MkdirOpts{ Mkparents: true, Flush: false, From 2106b4e3b117a048cd6e80006b1cc0b95757fe6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Mar 2019 15:24:49 +0100 Subject: [PATCH 3/3] coreapi mfs: gofmt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/coreapi.go | 2 +- core/coreapi/mfs.go | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index f6c0fe5a6..5811b3022 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -72,7 +72,7 @@ type CoreAPI struct { checkPublishAllowed func() error checkOnline func(allowOffline bool) error - filesRoot *mfs.Root // TODO: option filtering + filesRoot *mfs.Root // TODO: option filtering // ONLY for re-applying options in WithOptions, DO NOT USE ANYWHERE ELSE nd *core.IpfsNode diff --git a/core/coreapi/mfs.go b/core/coreapi/mfs.go index fa901ac73..a355c50f9 100644 --- a/core/coreapi/mfs.go +++ b/core/coreapi/mfs.go @@ -2,15 +2,15 @@ package coreapi import ( "context" + "errors" "fmt" - "github.com/ipfs/go-mfs" - "github.com/pkg/errors" "io" "os" gopath "path" "sync" "time" + "github.com/ipfs/go-mfs" coreiface "github.com/ipfs/interface-go-ipfs-core" ) @@ -19,7 +19,7 @@ const defaultDirPerm = 0755 type MfsAPI CoreAPI -type fileNodeInfo struct{ +type fileNodeInfo struct { l mfs.NodeListing } @@ -48,7 +48,7 @@ func (i *fileNodeInfo) Sys() interface{} { } func (api *MfsAPI) Create(ctx context.Context, path coreiface.MfsPath) (coreiface.File, error) { - return api.OpenFile(ctx, path, os.O_CREATE | os.O_WRONLY, defaultFilePerm) + return api.OpenFile(ctx, path, os.O_CREATE|os.O_WRONLY, defaultFilePerm) } func (api *MfsAPI) Open(ctx context.Context, path coreiface.MfsPath) (coreiface.File, error) { @@ -84,7 +84,6 @@ func (f *mfsFile) Close() error { return f.FileDescriptor.Close() } - func (f *mfsFile) ReadAt(p []byte, off int64) (int, error) { // TODO: implement in MFS with less locking f.lk.Lock() @@ -137,9 +136,9 @@ func (api *MfsAPI) OpenFile(ctx context.Context, path coreiface.MfsPath, flag in } flags := mfs.Flags{ - Read: flag & os.O_WRONLY == 0, - Write: flag & (os.O_WRONLY | os.O_RDWR) != 0, - Sync: flag & os.O_SYNC != 0, + Read: flag&os.O_WRONLY == 0, + Write: flag&(os.O_WRONLY|os.O_RDWR) != 0, + Sync: flag&os.O_SYNC != 0, } _, err = fn.Open(flags) @@ -250,8 +249,8 @@ func (api *MfsAPI) ReadDir(ctx context.Context, path coreiface.MfsPath) ([]os.Fi func (api *MfsAPI) MkdirAll(ctx context.Context, path coreiface.MfsPath, perm os.FileMode) error { return mfs.Mkdir(api.filesRoot, path.String(), mfs.MkdirOpts{ - Mkparents: true, - Flush: false, + Mkparents: true, + Flush: false, //CidBuilder: prefix, TODO }) }