Add config option to enable urlstore.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/builder.go

@@ -213,7 +213,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 	n.GCLocker = bstore.NewGCLocker()
 	n.Blockstore = bstore.NewGCBlockstore(cbs, n.GCLocker)
 
-	if conf.Experimental.FilestoreEnabled {
+	if conf.Experimental.FilestoreEnabled || conf.Experimental.UrlstoreEnabled {
 		// hash security
 		n.Filestore = filestore.NewFilestore(cbs, n.Repo.FileManager())
 		n.Blockstore = bstore.NewGCBlockstore(n.Filestore, n.GCLocker)

docs/experimental-features.md

@@ -17,6 +17,7 @@ the above issue.
 - [go-multiplex stream muxer](#go-multiplex-stream-muxer)
 - [Raw leaves for unixfs files](#raw-leaves-for-unixfs-files)
 - [ipfs filestore](#ipfs-filestore)
+- [ipfs urlstore](#ipfs-urlstore)
 - [BadgerDB datastore](#badger-datastore)
 - [Private Networks](#private-networks)
 - [ipfs p2p](#ipfs-p2p)
@@ -164,6 +165,26 @@ And then pass the `--nocopy` flag when running `ipfs add`
 
 ---
 
+## ipfs urlstore
+Allows ipfs to retrieve blocks contents via a url instead of storing it in the datastore
+
+### State
+experimental.
+
+### In Version
+???.
+
+### How to enable
+Modify your ipfs config:
+```
+ipfs config --json Experimental.UrlstoreEnabled true
+```
+
+### Road to being a real feature
+???.
+
+---
+
 ## Private Networks
 
 Allows ipfs to only connect to other peers who have a shared secret key.

filestore/filestore_test.go

@@ -23,6 +23,7 @@ func newTestFilestore(t *testing.T) (string, *Filestore) {
 		t.Fatal(err)
 	}
 	fm := NewFileManager(mds, testdir)
+	fm.AllowFiles = true
 
 	bs := blockstore.NewBlockstore(mds)
 	fstore := NewFilestore(bs, fm)

filestore/fsrefstore.go

@@ -29,8 +29,10 @@ var FilestorePrefix = ds.NewKey("filestore")
 // to the actual location of the block data in the filesystem
 // (a path and an offset).
 type FileManager struct {
-	ds   ds.Batching
-	root string
+	AllowFiles bool
+	AllowUrls  bool
+	ds         ds.Batching
+	root       string
 }
 
 // CorruptReferenceError implements the error interface.
@@ -52,7 +54,7 @@ func (c CorruptReferenceError) Error() string {
 // datastore and root. All FilestoreNodes paths are relative to the
 // root path given here, which is prepended for any operations.
 func NewFileManager(ds ds.Batching, root string) *FileManager {
-	return &FileManager{dsns.Wrap(ds, FilestorePrefix), root}
+	return &FileManager{ds: dsns.Wrap(ds, FilestorePrefix), root: root}
 }
 
 // AllKeysChan returns a channel from which to read the keys stored in
@@ -157,6 +159,10 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) {
 }
 
 func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
+	if !f.AllowFiles {
+		return nil, fmt.Errorf("filestore not enabled")
+	}
+
 	p := filepath.FromSlash(d.GetFilePath())
 	abspath := filepath.Join(f.root, p)
 
@@ -196,6 +202,9 @@ func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error)
 
 // reads and verifies the block from URL
 func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
+	if !f.AllowUrls {
+		return nil, fmt.Errorf("urlstore not enabled")
+	}
 
 	req, err := http.NewRequest("GET", d.GetFilePath(), nil)
 	if err != nil {
@@ -257,6 +266,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
 	var dobj pb.DataObj
 
 	if !IsURL(b.PosInfo.FullPath) {
+		if !f.AllowFiles {
+			return fmt.Errorf("filestore not enabled")
+		}
 		if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
 			return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
 		}
@@ -268,6 +280,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
 
 		dobj.FilePath = proto.String(filepath.ToSlash(p))
 	} else {
+		if !f.AllowUrls {
+			return fmt.Errorf("urlstore not enabled")
+		}
 		dobj.FilePath = proto.String(b.PosInfo.FullPath)
 	}
 	dobj.Offset = proto.Uint64(b.PosInfo.Offset)

repo/config/experiments.go

@@ -2,6 +2,7 @@ package config
 
 type Experiments struct {
 	FilestoreEnabled     bool
+	UrlstoreEnabled      bool
 	ShardingEnabled      bool
 	Libp2pStreamMounting bool
 }

repo/fsrepo/fsrepo.go

@@ -175,8 +175,10 @@ func open(repoPath string) (repo.Repo, error) {
 		return nil, err
 	}
 
-	if r.config.Experimental.FilestoreEnabled {
+	if r.config.Experimental.FilestoreEnabled || r.config.Experimental.UrlstoreEnabled {
 		r.filemgr = filestore.NewFileManager(r.ds, filepath.Dir(r.path))
+		r.filemgr.AllowFiles = r.config.Experimental.FilestoreEnabled
+		r.filemgr.AllowUrls = r.config.Experimental.UrlstoreEnabled
 	}
 
 	keepLocked = true