diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 3676ed891..ea6ccfc1d 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -56,6 +56,7 @@ const ( initProfileOptionKwd = "init-profile" ipfsMountKwd = "mount-ipfs" ipnsMountKwd = "mount-ipns" + mfsMountKwd = "mount-mfs" migrateKwd = "migrate" mountKwd = "mount" offlineKwd = "offline" // global option @@ -173,6 +174,7 @@ Headers. cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"), cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), + cmds.StringOption(mfsMountKwd, "Path to the mountpoint for MFS (if using --mount). Defaults to config setting."), cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow RPC API access to unlisted hashes"), cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"), cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"), @@ -1062,17 +1064,23 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { nsdir = cfg.Mounts.IPNS } + mfsdir, found := req.Options[mfsMountKwd].(string) + if !found { + mfsdir = cfg.Mounts.MFS + } + node, err := cctx.ConstructNode() if err != nil { return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) } - err = nodeMount.Mount(node, fsdir, nsdir) + err = nodeMount.Mount(node, fsdir, nsdir, mfsdir) if err != nil { return err } fmt.Printf("IPFS mounted at: %s\n", fsdir) fmt.Printf("IPNS mounted at: %s\n", nsdir) + fmt.Printf("MFS mounted at: %s\n", mfsdir) return nil } diff --git a/config/init.go b/config/init.go index a40efdead..40c24bd25 100644 --- a/config/init.go +++ b/config/init.go @@ -52,6 +52,7 @@ func InitWithIdentity(identity Identity) (*Config, error) { Mounts: Mounts{ IPFS: "/ipfs", IPNS: "/ipns", + MFS: "/mfs", }, Ipns: Ipns{ diff --git a/config/mounts.go b/config/mounts.go index dfdd1e5bf..571316cf3 100644 --- a/config/mounts.go +++ b/config/mounts.go @@ -4,5 +4,6 @@ package config type Mounts struct { IPFS string IPNS string + MFS string FuseAllowOther bool } diff --git a/core/commands/mount_nofuse.go b/core/commands/mount_nofuse.go index c425aff0f..103678e77 100644 --- a/core/commands/mount_nofuse.go +++ b/core/commands/mount_nofuse.go @@ -14,10 +14,11 @@ var MountCmd = &cmds.Command{ ShortDescription: ` This version of ipfs is compiled without fuse support, which is required for mounting. If you'd like to be able to mount, please use a version of -ipfs compiled with fuse. +Kubo compiled with fuse. For the latest instructions, please check the project's repository: - http://github.com/ipfs/go-ipfs + http://github.com/ipfs/kubo + https://github.com/ipfs/kubo/blob/master/docs/fuse.md `, }, } diff --git a/core/commands/mount_unix.go b/core/commands/mount_unix.go index 52a1b843b..6051f86aa 100644 --- a/core/commands/mount_unix.go +++ b/core/commands/mount_unix.go @@ -18,6 +18,7 @@ import ( const ( mountIPFSPathOptionName = "ipfs-path" mountIPNSPathOptionName = "ipns-path" + mountMFSPathOptionName = "mfs-path" ) var MountCmd = &cmds.Command{ @@ -25,14 +26,14 @@ var MountCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Mounts IPFS to the filesystem (read-only).", ShortDescription: ` -Mount IPFS at a read-only mountpoint on the OS (default: /ipfs and /ipns). +Mount IPFS at a read-only mountpoint on the OS (default: /ipfs, /ipns, /mfs). All IPFS objects will be accessible under that directory. Note that the root will not be listable, as it is virtual. Access known paths directly. You may have to create /ipfs and /ipns before using 'ipfs mount': -> sudo mkdir /ipfs /ipns -> sudo chown $(whoami) /ipfs /ipns +> sudo mkdir /ipfs /ipns /mfs +> sudo chown $(whoami) /ipfs /ipns /mfs > ipfs daemon & > ipfs mount `, @@ -44,8 +45,8 @@ root will not be listable, as it is virtual. Access known paths directly. You may have to create /ipfs and /ipns before using 'ipfs mount': -> sudo mkdir /ipfs /ipns -> sudo chown $(whoami) /ipfs /ipns +> sudo mkdir /ipfs /ipns /mfs +> sudo chown $(whoami) /ipfs /ipns /mfs > ipfs daemon & > ipfs mount @@ -67,6 +68,7 @@ baz > ipfs mount IPFS mounted at: /ipfs IPNS mounted at: /ipns +MFS mounted at: /mfs > cd /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC > ls bar @@ -81,6 +83,7 @@ baz Options: []cmds.Option{ cmds.StringOption(mountIPFSPathOptionName, "f", "The path where IPFS should be mounted."), cmds.StringOption(mountIPNSPathOptionName, "n", "The path where IPNS should be mounted."), + cmds.StringOption(mountMFSPathOptionName, "m", "The path where MFS should be mounted."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfg, err := env.(*oldcmds.Context).GetConfig() @@ -109,7 +112,12 @@ baz nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare! } - err = nodeMount.Mount(nd, fsdir, nsdir) + mfsdir, found := req.Options[mountMFSPathOptionName].(string) + if !found { + mfsdir = cfg.Mounts.MFS + } + + err = nodeMount.Mount(nd, fsdir, nsdir, mfsdir) if err != nil { return err } @@ -117,6 +125,7 @@ baz var output config.Mounts output.IPFS = fsdir output.IPNS = nsdir + output.MFS = mfsdir return cmds.EmitOnce(res, &output) }, Type: config.Mounts{}, @@ -124,6 +133,7 @@ baz cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, mounts *config.Mounts) error { fmt.Fprintf(w, "IPFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPFS)) fmt.Fprintf(w, "IPNS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPNS)) + fmt.Fprintf(w, "MFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.MFS)) return nil }), diff --git a/core/core.go b/core/core.go index 54c987527..3440895e7 100644 --- a/core/core.go +++ b/core/core.go @@ -134,6 +134,7 @@ type IpfsNode struct { type Mounts struct { Ipfs mount.Mount Ipns mount.Mount + Mfs mount.Mount } // Close calls Close() on the App object diff --git a/docs/changelogs/v0.35.md b/docs/changelogs/v0.35.md index 2826541cc..c5fccf7ad 100644 --- a/docs/changelogs/v0.35.md +++ b/docs/changelogs/v0.35.md @@ -12,6 +12,7 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [🔦 Highlights](#-highlights) - [Opt-in HTTP Retrieval client](#opt-in-http-retrieval-client) - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs) + - [Experimental support for MFS as a FUSE mount point](#experimental-support-for-mfs-as-a-fuse-mount-point) - [Grid view in WebUI](#grid-view-in-webui) - [Enhanced DAG-Shaping Controls](#enhanced-dag-shaping-controls) - [New DAG-Shaping `ipfs add` Options](#new-dag-shaping-ipfs-add-options) @@ -64,6 +65,14 @@ Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS al See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details. +#### Experimental support for MFS as a FUSE mount point + +The MFS root (filesystem behind the `ipfs files` API) is now available as a read/write FUSE mount point at `Mounts.MFS`. This filesystem is mounted in the same way as `Mounts.IPFS` and `Mounts.IPNS` when running `ipfs mount` or `ipfs daemon --mount`. + +Note that the operations supported by the MFS FUSE mountpoint are limited, since MFS doesn't store file attributes. + +See [`Mounts`](https://github.com/ipfs/kubo/blob/master/docs/config.md#mounts) and [`docs/fuse.md`](https://github.com/ipfs/kubo/blob/master/docs/fuse.md) for more details. + #### Grid view in WebUI The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen: diff --git a/docs/config.md b/docs/config.md index 91cefe8dc..3da157d19 100644 --- a/docs/config.md +++ b/docs/config.md @@ -97,6 +97,7 @@ config file at runtime. - [`Mounts`](#mounts) - [`Mounts.IPFS`](#mountsipfs) - [`Mounts.IPNS`](#mountsipns) + - [`Mounts.MFS`](#mountsmfs) - [`Mounts.FuseAllowOther`](#mountsfuseallowother) - [`Pinning`](#pinning) - [`Pinning.RemoteServices`](#pinningremoteservices) @@ -1368,7 +1369,8 @@ Default: `cache` ## `Mounts` -**EXPERIMENTAL:** read about current limitations at [fuse.md](./fuse.md). +> [!CAUTION] +> **EXPERIMENTAL:** read about current limitations at [fuse.md](./fuse.md). FUSE mount point configuration options. @@ -1388,6 +1390,18 @@ Default: `/ipns` Type: `string` (filesystem path) +### `Mounts.MFS` + +Mountpoint for Mutable File System (MFS) behind the `ipfs files` API. + +> [!CAUTION] +> - Write support is highly experimental and not recommended for mission-critical deployments. +> - Avoid storing lazy-loaded datasets in MFS. Exposing a partially local, lazy-loaded DAG risks operating system search indexers crawling it, which may trigger unintended network prefetching of non-local DAG components. + +Default: `/mfs` + +Type: `string` (filesystem path) + ### `Mounts.FuseAllowOther` Sets the 'FUSE allow other'-option on the mount point. diff --git a/docs/experimental-features.md b/docs/experimental-features.md index c0832b2e1..7d0069fc6 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -404,7 +404,7 @@ We also support the use of protocol names of the form /x/$NAME/http where $NAME ## FUSE -FUSE makes it possible to mount `/ipfs` and `/ipns` namespaces in your OS, +FUSE makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS using a subset of filesystem abstractions. It is considered EXPERIMENTAL due to limited (and buggy) support on some platforms. diff --git a/docs/fuse.md b/docs/fuse.md index b4b966e52..fde9307fd 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -2,7 +2,7 @@ **EXPERIMENTAL:** FUSE support is limited, YMMV. -Kubo makes it possible to mount `/ipfs` and `/ipns` namespaces in your OS, +Kubo makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS. ## Install FUSE @@ -50,18 +50,20 @@ speak with us, or if you figure something new out, please add to this document! ## Prepare mountpoints -By default ipfs uses `/ipfs` and `/ipns` directories for mounting, this can be -changed in config. You will have to create the `/ipfs` and `/ipns` directories +By default ipfs uses `/ipfs`, `/ipns` and `/mfs` directories for mounting, this can be +changed in config. You will have to create the `/ipfs`, `/ipns` and `/mfs` directories explicitly. Note that modifying root requires sudo permissions. ```sh # make the directories sudo mkdir /ipfs sudo mkdir /ipns +sudo mkdir /mfs # chown them so ipfs can use them without root permissions sudo chown /ipfs sudo chown /ipns +sudo chown /mfs ``` Depending on whether you are using OSX or Linux, follow the proceeding instructions. @@ -105,6 +107,25 @@ ipfs config --json Mounts.FuseAllowOther true ipfs daemon --mount ``` +## MFS mountpoint + +Kubo v0.35.0 and later supports mounting the MFS (Mutable File System) root as +a FUSE filesystem, enabling manipulation of content-addressed data like regular +files. The CID for any file or directory is retrievable via the `ipfs_cid` +extended attribute. + +```sh +getfattr -n ipfs_cid /mfs/welcome-to-IPFS.jpg +getfattr: Removing leading '/' from absolute path names +# file: mfs/welcome-to-IPFS.jpg +ipfs_cid="QmaeXDdwpUeKQcMy7d5SFBfVB4y7LtREbhm5KizawPsBSH" +``` + +Please note that the operations supported by the MFS FUSE mountpoint are +limited. Since the MFS wasn't designed to store file attributes like ownership +information, permissions and creation date, some applications like `vim` and +`sed` may misbehave due to missing functionality. + ## Troubleshooting #### `Permission denied` or `fusermount: user has no write access to mountpoint` error in Linux @@ -145,6 +166,7 @@ set for user running `ipfs mount` command. ``` sudo umount /ipfs sudo umount /ipns +sudo umount /mfs ``` #### Mounting fails with "error mounting: could not resolve name" diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go new file mode 100644 index 000000000..cedbe9967 --- /dev/null +++ b/fuse/mfs/mfs_test.go @@ -0,0 +1,342 @@ +//go:build !nofuse && !openbsd && !netbsd && !plan9 +// +build !nofuse,!openbsd,!netbsd,!plan9 + +package mfs + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + iofs "io/fs" + "os" + "slices" + "strconv" + "testing" + "time" + + "bazil.org/fuse" + "bazil.org/fuse/fs" + "bazil.org/fuse/fs/fstestutil" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/core/node" + "github.com/libp2p/go-libp2p-testing/ci" +) + +// Create an Ipfs.Node, a filesystem and a mount point. +func setUp(t *testing.T, ipfs *core.IpfsNode) (fs.FS, *fstestutil.Mount) { + if ci.NoFuse() { + t.Skip("Skipping FUSE tests") + } + + if ipfs == nil { + var err error + ipfs, err = core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + } + + fs := NewFileSystem(ipfs) + mnt, err := fstestutil.MountedT(t, fs, nil) + if err == fuse.ErrOSXFUSENotFound { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + + return fs, mnt +} + +// Test reading and writing a file. +func TestReadWrite(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + path := mnt.Dir + "/testrw" + content := make([]byte, 8196) + _, err := rand.Read(content) + if err != nil { + t.Fatal(err) + } + + t.Run("write", func(t *testing.T) { + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + buf := make([]byte, 8196) + l, err := f.Read(buf) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(content, buf[:l]) != true { + t.Fatal("read and write not equal") + } + }) +} + +// Test creating a directory. +func TestMkdir(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + path := mnt.Dir + "/foo/bar/baz/qux/quux" + + t.Run("write", func(t *testing.T) { + err := os.MkdirAll(path, iofs.ModeDir) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + stat, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if !stat.IsDir() { + t.Fatal("not dir") + } + }) +} + +// Test file persistence across mounts. +func TestPersistence(t *testing.T) { + ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + + content := make([]byte, 8196) + _, err = rand.Read(content) + if err != nil { + t.Fatal(err) + } + + t.Run("write", func(t *testing.T) { + _, mnt := setUp(t, ipfs) + defer mnt.Close() + path := mnt.Dir + "/testpersistence" + + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + _, mnt := setUp(t, ipfs) + defer mnt.Close() + path := mnt.Dir + "/testpersistence" + + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + buf := make([]byte, 8196) + l, err := f.Read(buf) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(content, buf[:l]) != true { + t.Fatal("read and write not equal") + } + }) +} + +// Test getting the file attributes. +func TestAttr(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + path := mnt.Dir + "/testattr" + content := make([]byte, 8196) + _, err := rand.Read(content) + if err != nil { + t.Fatal(err) + } + + t.Run("write", func(t *testing.T) { + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + if fi.IsDir() { + t.Fatal("file is a directory") + } + + if fi.ModTime().After(time.Now()) { + t.Fatal("future modtime") + } + if time.Since(fi.ModTime()) > time.Second { + t.Fatal("past modtime") + } + + if fi.Name() != "testattr" { + t.Fatal("invalid filename") + } + + if fi.Size() != 8196 { + t.Fatal("invalid size") + } + }) +} + +// Test concurrent access to the filesystem. +func TestConcurrentRW(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + files := 5 + fileWorkers := 5 + + path := mnt.Dir + "/testconcurrent" + content := make([][]byte, files) + + for i := range content { + content[i] = make([]byte, 8196) + _, err := rand.Read(content[i]) + if err != nil { + t.Fatal(err) + } + } + + t.Run("write", func(t *testing.T) { + errs := make(chan (error), 1) + for i := 0; i < files; i++ { + go func() { + var err error + defer func() { errs <- err }() + + f, err := os.Create(path + strconv.Itoa(i)) + if err != nil { + return + } + defer f.Close() + + _, err = f.Write(content[i]) + if err != nil { + return + } + }() + } + for i := 0; i < files; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } + }) + t.Run("read", func(t *testing.T) { + errs := make(chan (error), 1) + for i := 0; i < files*fileWorkers; i++ { + go func() { + var err error + defer func() { errs <- err }() + + f, err := os.Open(path + strconv.Itoa(i/fileWorkers)) + if err != nil { + return + } + defer f.Close() + + buf := make([]byte, 8196) + l, err := f.Read(buf) + if err != nil { + return + } + if bytes.Equal(content[i/fileWorkers], buf[:l]) != true { + err = errors.New("read and write not equal") + return + } + }() + } + for i := 0; i < files; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } + }) +} + +// Test ipfs_cid extended attribute +func TestMFSRootXattr(t *testing.T) { + ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + + fs, mnt := setUp(t, ipfs) + defer mnt.Close() + + node, err := fs.Root() + if err != nil { + t.Fatal(err) + } + + root := node.(*Dir) + + listReq := fuse.ListxattrRequest{} + listRes := fuse.ListxattrResponse{} + err = root.Listxattr(context.Background(), &listReq, &listRes) + if err != nil { + t.Fatal(err) + } + if slices.Compare(listRes.Xattr, []byte("ipfs_cid\x00")) != 0 { + t.Fatal("list xattr returns invalid value") + } + + getReq := fuse.GetxattrRequest{ + Name: "ipfs_cid", + } + getRes := fuse.GetxattrResponse{} + err = root.Getxattr(context.Background(), &getReq, &getRes) + if err != nil { + t.Fatal(err) + } + + ipldNode, err := ipfs.FilesRoot.GetDirectory().GetNode() + if err != nil { + t.Fatal(err) + } + + if slices.Compare(getRes.Xattr, []byte(ipldNode.Cid().String())) != 0 { + t.Fatal("xattr cid not equal to mfs root cid") + } +} diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go new file mode 100644 index 000000000..91cad257d --- /dev/null +++ b/fuse/mfs/mfs_unix.go @@ -0,0 +1,414 @@ +//go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse +// +build linux darwin freebsd netbsd openbsd +// +build !nofuse + +package mfs + +import ( + "context" + "io" + "os" + "sync" + "syscall" + "time" + + "bazil.org/fuse" + "bazil.org/fuse/fs" + + dag "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/boxo/mfs" + "github.com/ipfs/kubo/core" +) + +const ( + ipfsCIDXattr = "ipfs_cid" + mfsDirMode = os.ModeDir | 0755 + mfsFileMode = 0644 + blockSize = 512 + dirSize = 8 +) + +// FUSE filesystem mounted at /mfs. +type FileSystem struct { + root Dir +} + +// Get filesystem root. +func (fs *FileSystem) Root() (fs.Node, error) { + return &fs.root, nil +} + +// FUSE Adapter for MFS directories. +type Dir struct { + mfsDir *mfs.Directory +} + +// Directory attributes (stat). +func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { + attr.Mode = mfsDirMode + attr.Size = dirSize * blockSize + attr.Blocks = dirSize + return nil +} + +// Access files in a directory. +func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { + mfsNode, err := dir.mfsDir.Child(req.Name) + switch err { + case os.ErrNotExist: + return nil, syscall.Errno(syscall.ENOENT) + case nil: + default: + return nil, err + } + + switch mfsNode.Type() { + case mfs.TDir: + result := Dir{ + mfsDir: mfsNode.(*mfs.Directory), + } + return &result, nil + case mfs.TFile: + result := File{ + mfsFile: mfsNode.(*mfs.File), + } + return &result, nil + } + + return nil, syscall.Errno(syscall.ENOENT) +} + +// List (ls) MFS directory. +func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + var res []fuse.Dirent + nodes, err := dir.mfsDir.List(ctx) + if err != nil { + return nil, err + } + + for _, node := range nodes { + nodeType := fuse.DT_File + if node.Type == 1 { + nodeType = fuse.DT_Dir + } + res = append(res, fuse.Dirent{ + Type: nodeType, + Name: node.Name, + }) + } + return res, nil +} + +// Mkdir (mkdir) in MFS. +func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { + mfsDir, err := dir.mfsDir.Mkdir(req.Name) + if err != nil { + return nil, err + } + return &Dir{ + mfsDir: mfsDir, + }, nil +} + +// Remove (rm/rmdir) an MFS file. +func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + // Check for empty directory. + if req.Dir { + targetNode, err := dir.mfsDir.Child(req.Name) + if err != nil { + return err + } + target := targetNode.(*mfs.Directory) + + children, err := target.ListNames(ctx) + if err != nil { + return err + } + if len(children) > 0 { + return os.ErrExist + } + } + err := dir.mfsDir.Unlink(req.Name) + if err != nil { + return err + } + return dir.mfsDir.Flush() +} + +// Move (mv) an MFS file. +func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { + file, err := dir.mfsDir.Child(req.OldName) + if err != nil { + return err + } + node, err := file.GetNode() + if err != nil { + return err + } + targetDir := newDir.(*Dir) + + // Remove file if exists + err = targetDir.mfsDir.Unlink(req.NewName) + if err != nil && err != os.ErrNotExist { + return err + } + + err = targetDir.mfsDir.AddChild(req.NewName, node) + if err != nil { + return err + } + + err = dir.mfsDir.Unlink(req.OldName) + if err != nil { + return err + } + + return dir.mfsDir.Flush() +} + +// Create (touch) an MFS file. +func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + node := dag.NodeWithData(ft.FilePBData(nil, 0)) + if err := node.SetCidBuilder(dir.mfsDir.GetCidBuilder()); err != nil { + return nil, nil, err + } + + if err := dir.mfsDir.AddChild(req.Name, node); err != nil { + return nil, nil, err + } + + if err := dir.mfsDir.Flush(); err != nil { + return nil, nil, err + } + + mfsNode, err := dir.mfsDir.Child(req.Name) + if err != nil { + return nil, nil, err + } + if err := mfsNode.SetModTime(time.Now()); err != nil { + return nil, nil, err + } + + mfsFile := mfsNode.(*mfs.File) + + file := File{ + mfsFile: mfsFile, + } + + // Read access flags and create a handler. + accessMode := req.Flags & fuse.OpenAccessModeMask + flags := mfs.Flags{ + Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, + Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, + Sync: req.Flags|fuse.OpenSync > 0, + } + + fd, err := mfsFile.Open(flags) + if err != nil { + return nil, nil, err + } + handler := FileHandler{ + mfsFD: fd, + } + + return &file, &handler, nil +} + +// List dir xattr. +func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { + resp.Append(ipfsCIDXattr) + return nil +} + +// Get dir xattr. +func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + switch req.Name { + case ipfsCIDXattr: + node, err := dir.mfsDir.GetNode() + if err != nil { + return err + } + resp.Xattr = []byte(node.Cid().String()) + return nil + default: + return fuse.ErrNoXattr + } +} + +// FUSE adapter for MFS files. +type File struct { + mfsFile *mfs.File +} + +// File attributes. +func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { + size, _ := file.mfsFile.Size() + + attr.Size = uint64(size) + if size%blockSize == 0 { + attr.Blocks = uint64(size / blockSize) + } else { + attr.Blocks = uint64(size/blockSize + 1) + } + + mtime, _ := file.mfsFile.ModTime() + attr.Mtime = mtime + + attr.Mode = mfsFileMode + return nil +} + +// Open an MFS file. +func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + accessMode := req.Flags & fuse.OpenAccessModeMask + flags := mfs.Flags{ + Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, + Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, + Sync: req.Flags|fuse.OpenSync > 0, + } + fd, err := file.mfsFile.Open(flags) + if err != nil { + return nil, err + } + + if flags.Write { + if err := file.mfsFile.SetModTime(time.Now()); err != nil { + return nil, err + } + } + + return &FileHandler{ + mfsFD: fd, + }, nil +} + +// Sync the file's contents to MFS. +func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { + return file.mfsFile.Sync() +} + +// List file xattr. +func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { + resp.Append(ipfsCIDXattr) + return nil +} + +// Get file xattr. +func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + switch req.Name { + case ipfsCIDXattr: + node, err := file.mfsFile.GetNode() + if err != nil { + return err + } + resp.Xattr = []byte(node.Cid().String()) + return nil + default: + return fuse.ErrNoXattr + } +} + +// Wrapper for MFS's file descriptor that conforms to the FUSE fs.Handler +// interface. +type FileHandler struct { + mfsFD mfs.FileDescriptor + mu sync.Mutex +} + +// Read a opened MFS file. +func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) + if err != nil { + return err + } + + buf := make([]byte, req.Size) + l, err := fh.mfsFD.Read(buf) + + resp.Data = buf[:l] + + switch err { + case nil, io.EOF, io.ErrUnexpectedEOF: + return nil + default: + return err + } +} + +// Write writes to an opened MFS file. +func (fh *FileHandler) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + l, err := fh.mfsFD.WriteAt(req.Data, req.Offset) + if err != nil { + return err + } + resp.Size = l + + return nil +} + +// Flushes the file's buffer. +func (fh *FileHandler) Flush(ctx context.Context, req *fuse.FlushRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + return fh.mfsFD.Flush() +} + +// Closes the file. +func (fh *FileHandler) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + return fh.mfsFD.Close() +} + +// Create new filesystem. +func NewFileSystem(ipfs *core.IpfsNode) fs.FS { + return &FileSystem{ + root: Dir{ + mfsDir: ipfs.FilesRoot.GetDirectory(), + }, + } +} + +// Check that our structs implement all the interfaces we want. +type mfsDir interface { + fs.Node + fs.NodeGetxattrer + fs.NodeListxattrer + fs.HandleReadDirAller + fs.NodeRequestLookuper + fs.NodeMkdirer + fs.NodeRenamer + fs.NodeRemover + fs.NodeCreater +} + +var _ mfsDir = (*Dir)(nil) + +type mfsFile interface { + fs.Node + fs.NodeGetxattrer + fs.NodeListxattrer + fs.NodeOpener + fs.NodeFsyncer +} + +var _ mfsFile = (*File)(nil) + +type mfsHandler interface { + fs.Handle + fs.HandleReader + fs.HandleWriter + fs.HandleFlusher + fs.HandleReleaser +} + +var _ mfsHandler = (*FileHandler)(nil) diff --git a/fuse/mfs/mount_unix.go b/fuse/mfs/mount_unix.go new file mode 100644 index 000000000..7fe72e8df --- /dev/null +++ b/fuse/mfs/mount_unix.go @@ -0,0 +1,21 @@ +//go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse +// +build linux darwin freebsd netbsd openbsd +// +build !nofuse + +package mfs + +import ( + core "github.com/ipfs/kubo/core" + mount "github.com/ipfs/kubo/fuse/mount" +) + +// Mount mounts MFS at a given location, and returns a mount.Mount instance. +func Mount(ipfs *core.IpfsNode, mountpoint string) (mount.Mount, error) { + cfg, err := ipfs.Repo.Config() + if err != nil { + return nil, err + } + allowOther := cfg.Mounts.FuseAllowOther + fsys := NewFileSystem(ipfs) + return mount.NewMount(ipfs.Process, fsys, mountpoint, allowOther) +} diff --git a/fuse/node/mount_nofuse.go b/fuse/node/mount_nofuse.go index e6f512f8e..7423cb24d 100644 --- a/fuse/node/mount_nofuse.go +++ b/fuse/node/mount_nofuse.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("not compiled in") } diff --git a/fuse/node/mount_notsupp.go b/fuse/node/mount_notsupp.go index e9762a3e4..79ac0e791 100644 --- a/fuse/node/mount_notsupp.go +++ b/fuse/node/mount_notsupp.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("FUSE not supported on OpenBSD or NetBSD. See #5334 (https://github.com/ipfs/kubo/issues/5334).") } diff --git a/fuse/node/mount_test.go b/fuse/node/mount_test.go index 178fddcf6..1947f759f 100644 --- a/fuse/node/mount_test.go +++ b/fuse/node/mount_test.go @@ -56,10 +56,12 @@ func TestExternalUnmount(t *testing.T) { ipfsDir := dir + "/ipfs" ipnsDir := dir + "/ipns" + mfsDir := dir + "/mfs" mkdir(t, ipfsDir) mkdir(t, ipnsDir) + mkdir(t, mfsDir) - err = Mount(node, ipfsDir, ipnsDir) + err = Mount(node, ipfsDir, ipnsDir, mfsDir) if err != nil { if strings.Contains(err.Error(), "unable to check fuse version") || err == fuse.ErrOSXFUSENotFound { t.Skip(err) diff --git a/fuse/node/mount_unix.go b/fuse/node/mount_unix.go index a5a2a3716..9846d7a42 100644 --- a/fuse/node/mount_unix.go +++ b/fuse/node/mount_unix.go @@ -11,6 +11,7 @@ import ( core "github.com/ipfs/kubo/core" ipns "github.com/ipfs/kubo/fuse/ipns" + mfs "github.com/ipfs/kubo/fuse/mfs" mount "github.com/ipfs/kubo/fuse/mount" rofs "github.com/ipfs/kubo/fuse/readonly" @@ -31,7 +32,7 @@ var platformFuseChecks = func(*core.IpfsNode) error { return nil } -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // check if we already have live mounts. // if the user said "Mount", then there must be something wrong. // so, close them and try again. @@ -43,15 +44,19 @@ func Mount(node *core.IpfsNode, fsdir, nsdir string) error { // best effort _ = node.Mounts.Ipns.Unmount() } + if node.Mounts.Mfs != nil && node.Mounts.Mfs.IsActive() { + // best effort + _ = node.Mounts.Mfs.Unmount() + } if err := platformFuseChecks(node); err != nil { return err } - return doMount(node, fsdir, nsdir) + return doMount(node, fsdir, nsdir, mfsdir) } -func doMount(node *core.IpfsNode, fsdir, nsdir string) error { +func doMount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { fmtFuseErr := func(err error, mountpoint string) error { s := err.Error() if strings.Contains(s, fuseNoDirectory) { @@ -67,8 +72,8 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { } // this sync stuff is so that both can be mounted simultaneously. - var fsmount, nsmount mount.Mount - var err1, err2 error + var fsmount, nsmount, mfmount mount.Mount + var err1, err2, err3 error var wg sync.WaitGroup @@ -86,6 +91,12 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { }() } + wg.Add(1) + go func() { + defer wg.Done() + mfmount, err3 = mfs.Mount(node, mfsdir) + }() + wg.Wait() if err1 != nil { @@ -96,22 +107,33 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { log.Errorf("error mounting IPNS %s for IPFS %s: %s", nsdir, fsdir, err2) } - if err1 != nil || err2 != nil { + if err3 != nil { + log.Errorf("error mounting MFS %s: %s", mfsdir, err3) + } + + if err1 != nil || err2 != nil || err3 != nil { if fsmount != nil { _ = fsmount.Unmount() } if nsmount != nil { _ = nsmount.Unmount() } + if mfmount != nil { + _ = mfmount.Unmount() + } if err1 != nil { return fmtFuseErr(err1, fsdir) } - return fmtFuseErr(err2, nsdir) + if err2 != nil { + return fmtFuseErr(err2, nsdir) + } + return fmtFuseErr(err3, mfsdir) } // setup node state, so that it can be canceled node.Mounts.Ipfs = fsmount node.Mounts.Ipns = nsmount + node.Mounts.Mfs = mfmount return nil } diff --git a/fuse/node/mount_windows.go b/fuse/node/mount_windows.go index 33393f99a..42e6bc10b 100644 --- a/fuse/node/mount_windows.go +++ b/fuse/node/mount_windows.go @@ -4,7 +4,7 @@ import ( "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // TODO // currently a no-op, but we don't want to return an error return nil diff --git a/test/3nodetest/bootstrap/config b/test/3nodetest/bootstrap/config index ac441a19f..e22f25e90 100644 --- a/test/3nodetest/bootstrap/config +++ b/test/3nodetest/bootstrap/config @@ -15,7 +15,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "Current": "0.1.7", diff --git a/test/3nodetest/client/config b/test/3nodetest/client/config index 86ef0668d..fa8f923d5 100644 --- a/test/3nodetest/client/config +++ b/test/3nodetest/client/config @@ -17,7 +17,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", diff --git a/test/3nodetest/server/config b/test/3nodetest/server/config index fb16a6d7a..1e9db2a63 100644 --- a/test/3nodetest/server/config +++ b/test/3nodetest/server/config @@ -17,7 +17,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", diff --git a/test/cli/bitswap_config_test.go b/test/cli/bitswap_config_test.go index db7656ef2..9674d3cb6 100644 --- a/test/cli/bitswap_config_test.go +++ b/test/cli/bitswap_config_test.go @@ -167,7 +167,7 @@ func TestBitswapConfig(t *testing.T) { node.UpdateConfig(func(cfg *config.Config) { cfg.HTTPRetrieval.Enabled = config.False cfg.Bitswap.Libp2pEnabled = config.False - cfg.Bitswap.ServerEnabled = config.True // bad user config: cant enable server when libp2p is down + cfg.Bitswap.ServerEnabled = config.True // bad user config: can't enable server when libp2p is down }) res := node.RunIPFS("daemon") assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap") diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 12a1f71be..f9292cb23 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -206,9 +206,10 @@ test_init_ipfs() { ' test_expect_success "prepare config -- mounting" ' - mkdir mountdir ipfs ipns && + mkdir mountdir ipfs ipns mfs && test_config_set Mounts.IPFS "$(pwd)/ipfs" && - test_config_set Mounts.IPNS "$(pwd)/ipns" || + test_config_set Mounts.IPNS "$(pwd)/ipns" && + test_config_set Mounts.MFS "$(pwd)/mfs" || test_fsh cat "\"$IPFS_PATH/config\"" ' @@ -321,12 +322,14 @@ test_mount_ipfs() { test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && + do_umount "$(pwd)/mfs" || true && ipfs mount >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && + echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' diff --git a/test/sharness/t0030-mount.sh b/test/sharness/t0030-mount.sh index 0c0983d0c..6df7a26bb 100755 --- a/test/sharness/t0030-mount.sh +++ b/test/sharness/t0030-mount.sh @@ -16,7 +16,8 @@ if ! test_have_prereq FUSE; then fi -export IPFS_NS_MAP="welcome.example.com:/ipfs/$HASH_WELCOME_DOCS" +# echo -n "ipfs" > expected && ipfs add --cid-version 1 -Q -w expected +export IPFS_NS_MAP="welcome.example.com:/ipfs/bafybeicq7bvn5lz42qlmghaoiwrve74pzi53auqetbantp5kajucsabike" # start iptb + wait for peering NUM_NODES=5 @@ -27,17 +28,17 @@ startup_cluster $NUM_NODES # test mount failure before mounting properly. test_expect_success "'ipfs mount' fails when there is no mount dir" ' - tmp_ipfs_mount() { ipfsi 0 mount -f=not_ipfs -n=not_ipns >output 2>output.err; } && + tmp_ipfs_mount() { ipfsi 0 mount -f=not_ipfs -n=not_ipns -m=not_mfs >output 2>output.err; } && test_must_fail tmp_ipfs_mount ' test_expect_success "'ipfs mount' output looks good" ' test_must_be_empty output && - test_should_contain "not_ipns\|not_ipfs" output.err + test_should_contain "not_ipns\|not_ipfs\|not_mfs" output.err ' test_expect_success "setup and publish default IPNS value" ' - mkdir "$(pwd)/ipfs" "$(pwd)/ipns" && + mkdir "$(pwd)/ipfs" "$(pwd)/ipns" "$(pwd)/mfs" && ipfsi 0 name publish QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn ' @@ -46,12 +47,14 @@ test_expect_success "setup and publish default IPNS value" ' test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && - ipfsi 0 mount -f "$(pwd)/ipfs" -n "$(pwd)/ipns" >actual + do_umount "$(pwd)/mfs" || true && + ipfsi 0 mount -f "$(pwd)/ipfs" -n "$(pwd)/ipns" -m "$(pwd)/mfs" >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && + echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' @@ -63,21 +66,64 @@ test_expect_success FUSE "local symlink works" ' test_expect_success FUSE "can resolve ipns names" ' echo -n "ipfs" > expected && - cat ipns/welcome.example.com/ping > actual && + ipfsi 0 add --cid-version 1 -Q -w expected && + cat ipns/welcome.example.com/expected > actual && test_cmp expected actual ' +test_expect_success FUSE "create mfs file via fuse" ' + touch mfs/testfile && + ipfsi 0 files ls | grep testfile +' + +test_expect_success FUSE "create mfs dir via fuse" ' + mkdir mfs/testdir && + ipfsi 0 files ls | grep testdir +' + +test_expect_success FUSE "read mfs file from fuse" ' + echo content > mfs/testfile && + getfattr -n ipfs_cid mfs/testfile +' +test_expect_success FUSE "ipfs add file and read it back via fuse" ' + echo content3 | ipfsi 0 files write -e /testfile3 && + grep content3 mfs/testfile3 +' + +test_expect_success FUSE "ipfs add file and read it back via fuse" ' + echo content > testfile2 && + ipfsi 0 add --to-files /testfile2 testfile2 && + grep content mfs/testfile2 +' + +test_expect_success FUSE "test file xattr" ' + echo content > mfs/testfile && + getfattr -n ipfs_cid mfs/testfile +' + +test_expect_success FUSE "test file removal" ' + touch mfs/testfile && + rm mfs/testfile +' + +test_expect_success FUSE "test nested dirs" ' + mkdir -p mfs/foo/bar/baz/qux && + echo content > mfs/foo/bar/baz/qux/quux && + ipfsi 0 files stat /foo/bar/baz/qux/quux +' + test_expect_success "mount directories cannot be removed while active" ' - test_must_fail rmdir ipfs ipns 2>/dev/null + test_must_fail rmdir ipfs ipns mfs 2>/dev/null ' test_expect_success "unmount directories" ' do_umount "$(pwd)/ipfs" && - do_umount "$(pwd)/ipns" + do_umount "$(pwd)/ipns" && + do_umount "$(pwd)/mfs" ' test_expect_success "mount directories can be removed after shutdown" ' - rmdir ipfs ipns + rmdir ipfs ipns mfs ' test_expect_success 'stop iptb' ' diff --git a/test/sharness/t0270-filestore.sh b/test/sharness/t0270-filestore.sh index f2f63b0de..fc377c2d2 100755 --- a/test/sharness/t0270-filestore.sh +++ b/test/sharness/t0270-filestore.sh @@ -63,7 +63,7 @@ test_filestore_adds() { init_ipfs_filestore() { test_expect_success "clean up old node" ' - rm -rf "$IPFS_PATH" mountdir ipfs ipns + rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs diff --git a/test/sharness/t0271-filestore-utils.sh b/test/sharness/t0271-filestore-utils.sh index e7c11646c..5fd335659 100755 --- a/test/sharness/t0271-filestore-utils.sh +++ b/test/sharness/t0271-filestore-utils.sh @@ -10,7 +10,7 @@ test_description="Test out the filestore nocopy functionality" test_init_filestore() { test_expect_success "clean up old node" ' - rm -rf "$IPFS_PATH" mountdir ipfs ipns + rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs