diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index fa25eb5d7..9d4bb51a4 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -85,7 +85,7 @@ environment variable: f := req.Files if f != nil { - confFile, err := f.NextFile() + _, confFile, err := f.NextFile() if err != nil { return err } diff --git a/core/commands/add.go b/core/commands/add.go index d78bad323..058519322 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,9 +6,9 @@ import ( "os" "strings" - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" - options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb" files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files" @@ -226,24 +226,17 @@ You can now check what blocks have been created by: outChan := make(chan interface{}) req := res.Request() - sizeFile, ok := req.Files.(files.SizeFile) - if ok { - // Could be slow. - go func() { - size, err := sizeFile.Size() - if err != nil { - log.Warningf("error getting files size: %s", err) - // see comment above - return - } + // Could be slow. + go func() { + size, err := req.Files.Size() + if err != nil { + log.Warningf("error getting files size: %s", err) + // see comment above + return + } - sizeChan <- size - }() - } else { - // we don't need to error, the progress bar just - // won't know how big the files are - log.Warning("cannot determine size of input file") - } + sizeChan <- size + }() progressBar := func(wait chan struct{}) { defer close(wait) diff --git a/core/commands/block.go b/core/commands/block.go index 142626712..054e621d8 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -153,7 +153,7 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1. return err } - file, err := req.Files.NextFile() + _, file, err := req.Files.NextFile() if err != nil { return err } diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index e9bed7ad7..1e3b0441e 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -93,7 +93,7 @@ into an object of the specified format. } for { - file, err := req.Files.NextFile() + _, file, err := req.Files.NextFile() if err == io.EOF { // Finished the list of files. break diff --git a/core/commands/files.go b/core/commands/files.go index d262ee213..eeff856a7 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -769,7 +769,7 @@ stat' on the file or any of its ancestors. return err } - input, err := req.Files.NextFile() + _, input, err := req.Files.NextFile() if err != nil { return err } diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 0843c4e2f..bba3abae5 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -391,7 +391,7 @@ And then run: return err } - input, err := req.Files.NextFile() + _, input, err := req.Files.NextFile() if err != nil && err != io.EOF { return err } diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index aa022db5a..0e8fd8d11 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -60,7 +60,7 @@ the limit will not be respected by the network. return err } - data, err := req.Files.NextFile() + _, data, err := req.Files.NextFile() if err != nil { return err } @@ -107,7 +107,7 @@ Example: return err } - data, err := req.Files.NextFile() + _, data, err := req.Files.NextFile() if err != nil { return err } diff --git a/core/commands/tar.go b/core/commands/tar.go index 2a98494e2..85e71b3de 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -44,7 +44,7 @@ represent it. return err } - fi, err := req.Files.NextFile() + name, fi, err := req.Files.NextFile() if err != nil { return err } @@ -56,9 +56,8 @@ represent it. c := node.Cid() - fi.FileName() return cmds.EmitOnce(res, &coreiface.AddEvent{ - Name: fi.FileName(), + Name: name, Hash: c.String(), }) }, diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 3a214085c..773b36dc0 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -2,9 +2,8 @@ package iface import ( "context" - "io" - options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files" ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" @@ -18,11 +17,6 @@ type AddEvent struct { Size string `json:",omitempty"` } -type UnixfsFile interface { - files.SizeFile - io.Seeker -} - // UnixfsAPI is the basic interface to immutable files in IPFS // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { @@ -35,7 +29,7 @@ type UnixfsAPI interface { // // Note that some implementations of this API may apply the specified context // to operations performed on the returned file - Get(context.Context, Path) (UnixfsFile, error) + Get(context.Context, Path) (files.File, error) // Ls returns the list of links in a directory Ls(context.Context, Path) ([]*ipld.Link, error) diff --git a/core/coreapi/name_test.go b/core/coreapi/name_test.go index 1b64508f5..85049b8b8 100644 --- a/core/coreapi/name_test.go +++ b/core/coreapi/name_test.go @@ -20,7 +20,7 @@ import ( var rnd = rand.New(rand.NewSource(0x62796532303137)) func addTestObject(ctx context.Context, api coreiface.CoreAPI) (coreiface.Path, error) { - return api.Unixfs().Add(ctx, files.NewReaderFile("", "", ioutil.NopCloser(&io.LimitedReader{R: rnd, N: 4092}), nil)) + return api.Unixfs().Add(ctx, files.NewReaderFile(ioutil.NopCloser(&io.LimitedReader{R: rnd, N: 4092}), nil)) } func appendPath(p coreiface.Path, sub string) coreiface.Path { diff --git a/core/coreapi/unixfile.go b/core/coreapi/unixfile.go index 1ff44e3bf..d2ef89887 100644 --- a/core/coreapi/unixfile.go +++ b/core/coreapi/unixfile.go @@ -4,11 +4,6 @@ import ( "context" "errors" "io" - "os" - gopath "path" - "time" - - "github.com/ipfs/go-ipfs/core/coreapi/interface" files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files" ft "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs" @@ -23,44 +18,11 @@ const prefetchFiles = 4 // TODO: this probably belongs in go-unixfs (and could probably replace a chunk of it's interface in the long run) -type sizeInfo struct { - size int64 - name string - modTime time.Time -} - -func (s *sizeInfo) Name() string { - return s.name -} - -func (s *sizeInfo) Size() int64 { - return s.size -} - -func (s *sizeInfo) Mode() os.FileMode { - return 0444 // all read -} - -func (s *sizeInfo) ModTime() time.Time { - return s.modTime -} - -func (s *sizeInfo) IsDir() bool { - return false -} - -func (s *sizeInfo) Sys() interface{} { - return nil -} - type ufsDirectory struct { ctx context.Context dserv ipld.DAGService files chan *ipld.Link - - name string - path string } func (d *ufsDirectory) Close() error { @@ -71,30 +33,23 @@ func (d *ufsDirectory) Read(_ []byte) (int, error) { return 0, files.ErrNotReader } -func (d *ufsDirectory) FileName() string { - return d.name -} - -func (d *ufsDirectory) FullPath() string { - return d.path -} - func (d *ufsDirectory) IsDirectory() bool { return true } -func (d *ufsDirectory) NextFile() (files.File, error) { +func (d *ufsDirectory) NextFile() (string, files.File, error) { l, ok := <-d.files if !ok { - return nil, io.EOF + return "", nil, io.EOF } nd, err := l.GetNode(d.ctx, d.dserv) if err != nil { - return nil, err + return "", nil, err } - return newUnixfsFile(d.ctx, d.dserv, nd, l.Name, d) + f, err := newUnixfsFile(d.ctx, d.dserv, nd, d) + return l.Name, f, err } func (d *ufsDirectory) Size() (int64, error) { @@ -107,32 +62,21 @@ func (d *ufsDirectory) Seek(offset int64, whence int) (int64, error) { type ufsFile struct { uio.DagReader - - name string - path string } func (f *ufsFile) IsDirectory() bool { return false } -func (f *ufsFile) NextFile() (files.File, error) { - return nil, files.ErrNotDirectory -} - -func (f *ufsFile) FileName() string { - return f.name -} - -func (f *ufsFile) FullPath() string { - return f.path +func (f *ufsFile) NextFile() (string, files.File, error) { + return "", nil, files.ErrNotDirectory } func (f *ufsFile) Size() (int64, error) { return int64(f.DagReader.Size()), nil } -func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, path string) (iface.UnixfsFile, error) { +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.File, error) { dir, err := uio.NewDirectoryFromNode(dserv, nd) if err != nil { return nil, err @@ -157,18 +101,10 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name dserv: dserv, files: fileCh, - - name: name, - path: path, }, nil } -func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, parent files.File) (iface.UnixfsFile, error) { - path := name - if parent != nil { - path = gopath.Join(parent.FullPath(), name) - } - +func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, parent files.File) (files.File, error) { switch dn := nd.(type) { case *dag.ProtoNode: fsn, err := ft.FSNodeFromBytes(dn.Data()) @@ -176,7 +112,7 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, nam return nil, err } if fsn.IsDir() { - return newUnixfsDir(ctx, dserv, nd, name, path) + return newUnixfsDir(ctx, dserv, nd) } case *dag.RawNode: @@ -191,10 +127,5 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, nam return &ufsFile{ DagReader: dr, - - name: name, - path: path, }, nil } - -var _ os.FileInfo = &sizeInfo{} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 4f129d6e0..f3c5038ca 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -133,7 +133,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options return coreiface.IpfsPath(nd.Cid()), nil } -func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (coreiface.UnixfsFile, error) { +func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.File, error) { ses := api.core().getSession(ctx) nd, err := ses.ResolveNode(ctx, p) @@ -141,7 +141,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (coreiface.Unix return nil, err } - return newUnixfsFile(ctx, ses.dag, nd, "", nil) + return newUnixfsFile(ctx, ses.dag, nd, nil) } // Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format: diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 228c04a05..38b91036e 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -8,7 +8,6 @@ import ( "io" "io/ioutil" "math" - "os" "strconv" "strings" "sync" @@ -136,33 +135,36 @@ func makeAPI(ctx context.Context) (*core.IpfsNode, coreiface.CoreAPI, error) { func strFile(data string) func() files.File { return func() files.File { - return files.NewReaderFile("", "", ioutil.NopCloser(strings.NewReader(data)), nil) + return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(data)), nil) } } func twoLevelDir() func() files.File { return func() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewSliceFile("t/abc", "t/abc", []files.File{ - files.NewReaderFile("t/abc/def", "t/abc/def", ioutil.NopCloser(strings.NewReader("world")), nil), - }), - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + return files.NewSliceFile([]files.FileEntry{{ + Name: "abc", File: files.NewSliceFile([]files.FileEntry{ + {Name: "def", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("world")), nil)}, + })}, + + {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)}, }) } } func flatDir() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + return files.NewSliceFile([]files.FileEntry{ + {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)}, }) } -func wrapped(f files.File) files.File { - return files.NewSliceFile("", "", []files.File{ - f, - }) +func wrapped(name string) func(f files.File) files.File { + return func(f files.File) files.File { + return files.NewSliceFile([]files.FileEntry{ + {Name: name, File: f}, + }) + } } func TestAdd(t *testing.T) { @@ -180,7 +182,7 @@ func TestAdd(t *testing.T) { path string err string - recursive bool + wrap string events []coreiface.AddEvent @@ -277,36 +279,36 @@ func TestAdd(t *testing.T) { }, // multi file { - name: "simpleDir", - data: flatDir, - recursive: true, - path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + name: "simpleDir", + data: flatDir, + wrap: "t", + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", }, { - name: "twoLevelDir", - data: twoLevelDir(), - recursive: true, - path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + name: "twoLevelDir", + data: twoLevelDir(), + wrap: "t", + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", }, // wrapped { name: "addWrapped", path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE", data: func() files.File { - return files.NewReaderFile("foo", "foo", ioutil.NopCloser(strings.NewReader(helloStr)), nil) + return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, - expect: wrapped, + expect: wrapped("foo"), opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, { name: "stdinWrapped", path: "/ipfs/QmU3r81oZycjHS9oaSHw37ootMFuFUw1DvMLKXPsezdtqU", data: func() files.File { - return files.NewReaderFile("", os.Stdin.Name(), ioutil.NopCloser(strings.NewReader(helloStr)), nil) + return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, expect: func(files.File) files.File { - return files.NewSliceFile("", "", []files.File{ - files.NewReaderFile("QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", ioutil.NopCloser(strings.NewReader(helloStr)), nil), + return files.NewSliceFile([]files.FileEntry{ + {Name: "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)}, }) }, opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, @@ -315,68 +317,68 @@ func TestAdd(t *testing.T) { name: "stdinNamed", path: "/ipfs/QmQ6cGBmb3ZbdrQW1MRm1RJnYnaxCqfssz7CrTa9NEhQyS", data: func() files.File { - return files.NewReaderFile("", os.Stdin.Name(), ioutil.NopCloser(strings.NewReader(helloStr)), nil) + return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, expect: func(files.File) files.File { - return files.NewSliceFile("", "", []files.File{ - files.NewReaderFile("test", "test", ioutil.NopCloser(strings.NewReader(helloStr)), nil), + return files.NewSliceFile([]files.FileEntry{ + {Name: "test", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)}, }) }, opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")}, }, { - name: "twoLevelDirWrapped", - data: twoLevelDir(), - recursive: true, - expect: wrapped, - path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", - opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + name: "twoLevelDirWrapped", + data: twoLevelDir(), + wrap: "t", + expect: wrapped(""), + path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, { - name: "twoLevelInlineHash", - data: twoLevelDir(), - recursive: true, - expect: wrapped, - path: "/ipfs/zBunoruKoyCHKkALNSWxDvj4L7yuQnMgQ4hUa9j1Z64tVcDEcu6Zdetyu7eeFCxMPfxb7YJvHeFHoFoHMkBUQf6vfdhmi", - opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true), options.Unixfs.Hash(mh.SHA3)}, + name: "twoLevelInlineHash", + data: twoLevelDir(), + wrap: "t", + expect: wrapped(""), + path: "/ipfs/zBunoruKoyCHKkALNSWxDvj4L7yuQnMgQ4hUa9j1Z64tVcDEcu6Zdetyu7eeFCxMPfxb7YJvHeFHoFoHMkBUQf6vfdhmi", + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true), options.Unixfs.Hash(mh.SHA3)}, }, // hidden { name: "hiddenFiles", data: func() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewReaderFile("t/.bar", "t/.bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + return files.NewSliceFile([]files.FileEntry{ + {Name: ".bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)}, }) }, - recursive: true, - path: "/ipfs/QmehGvpf2hY196MzDFmjL8Wy27S4jbgGDUAhBJyvXAwr3g", - opts: []options.UnixfsAddOption{options.Unixfs.Hidden(true)}, + wrap: "t", + path: "/ipfs/QmehGvpf2hY196MzDFmjL8Wy27S4jbgGDUAhBJyvXAwr3g", + opts: []options.UnixfsAddOption{options.Unixfs.Hidden(true)}, }, { name: "hiddenFileAlwaysAdded", data: func() files.File { - return files.NewReaderFile(".foo", ".foo", ioutil.NopCloser(strings.NewReader(helloStr)), nil) + return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, - recursive: true, - path: hello, + wrap: ".foo", + path: hello, }, { name: "hiddenFilesNotAdded", data: func() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewReaderFile("t/.bar", "t/.bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + return files.NewSliceFile([]files.FileEntry{ + {Name: ".bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)}, + {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)}, }) }, expect: func(files.File) files.File { return flatDir() }, - recursive: true, - path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", - opts: []options.UnixfsAddOption{options.Unixfs.Hidden(false)}, + wrap: "t", + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + opts: []options.UnixfsAddOption{options.Unixfs.Hidden(false)}, }, // Events / Progress { @@ -396,8 +398,8 @@ func TestAdd(t *testing.T) { {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, }, - recursive: true, - opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, + wrap: "t", + opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, }, { name: "dirAddEvents", @@ -410,13 +412,13 @@ func TestAdd(t *testing.T) { {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, }, - recursive: true, + wrap: "t", }, { name: "progress1M", data: func() files.File { r := bytes.NewReader(bytes.Repeat([]byte{0}, 1000000)) - return files.NewReaderFile("", "", ioutil.NopCloser(r), nil) + return files.NewReaderFile(ioutil.NopCloser(r), nil) }, path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", events: []coreiface.AddEvent{ @@ -426,8 +428,8 @@ func TestAdd(t *testing.T) { {Name: "", Bytes: 1000000}, {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Size: "1000256"}, }, - recursive: true, - opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, + wrap: "t", + opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, }, } @@ -439,9 +441,9 @@ func TestAdd(t *testing.T) { // recursive logic data := testCase.data() - if testCase.recursive { - data = files.NewSliceFile("", "", []files.File{ - data, + if testCase.wrap != "" { + data = files.NewSliceFile([]files.FileEntry{ + {Name: testCase.wrap, File: data}, }) } @@ -516,12 +518,16 @@ func TestAdd(t *testing.T) { // compare file structure with Unixfs().Get - var cmpFile func(orig files.File, got files.File) - cmpFile = func(orig files.File, got files.File) { + var cmpFile func(origName string, orig files.File, gotName string, got files.File) + cmpFile = func(origName string, orig files.File, gotName string, got files.File) { if orig.IsDirectory() != got.IsDirectory() { t.Fatal("file type mismatch") } + if origName != gotName { + t.Fatal("file name mismatch") + } + if !orig.IsDirectory() { defer orig.Close() defer got.Close() @@ -544,8 +550,8 @@ func TestAdd(t *testing.T) { } for { - fo, err := orig.NextFile() - fg, err2 := got.NextFile() + origName, origFile, err := orig.NextFile() + gotName, gotFile, err2 := got.NextFile() if err != nil { if err == io.EOF && err2 == io.EOF { @@ -557,7 +563,7 @@ func TestAdd(t *testing.T) { t.Fatal(err) } - cmpFile(fo, fg) + cmpFile(origName, origFile, gotName, gotFile) } } @@ -571,7 +577,7 @@ func TestAdd(t *testing.T) { orig = testCase.expect(orig) } - cmpFile(orig, f) + cmpFile("", orig, "", f) }) } } diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 3379689f4..103916c77 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -386,7 +386,7 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, nam } func (i *gatewayHandler) postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { - p, err := i.api.Unixfs().Add(ctx, files.NewReaderFile("", "", ioutil.NopCloser(r.Body), nil)) + p, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(ioutil.NopCloser(r.Body), nil)) if err != nil { internalWebError(w, err) return diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 868bd5d7e..fcfa4aa6e 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -311,7 +311,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - f, err := files.NewSerialFile(filepath.Base(root), root, false, stat) + f, err := files.NewSerialFile(root, false, stat) if err != nil { return "", err } @@ -322,7 +322,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - err = fileAdder.addFile(f) + err = fileAdder.addFile(filepath.Base(root), f) if err != nil { return "", err } @@ -340,7 +340,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { // Returns the path of the added file ("/filename"), the DAG node of // the directory, and and error if any. func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, ipld.Node, error) { - file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil) + file := files.NewReaderFile(ioutil.NopCloser(r), nil) fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) if err != nil { return "", nil, err @@ -349,7 +349,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, ipld.No defer n.Blockstore.PinLock().Unlock() - err = fileAdder.addFile(file) + err = fileAdder.addFile(filename, file) if err != nil { return "", nil, err } @@ -416,20 +416,20 @@ func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) { // single files.File f is treated as a directory, affecting hidden file // semantics. for { - f, err := file.NextFile() + name, f, err := file.NextFile() if err == io.EOF { // Finished the list of files. break } else if err != nil { return nil, err } - if err := adder.addFile(f); err != nil { + if err := adder.addFile(name, f); err != nil { return nil, err } } break default: - if err := adder.addFile(file); err != nil { + if err := adder.addFile("", file); err != nil { return nil, err } break @@ -447,7 +447,7 @@ func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) { return nd, adder.PinRoot() } -func (adder *Adder) addFile(file files.File) error { +func (adder *Adder) addFile(path string, file files.File) error { err := adder.maybePauseForGC() if err != nil { return err @@ -468,7 +468,7 @@ func (adder *Adder) addFile(file files.File) error { adder.liveNodes++ if file.IsDirectory() { - return adder.addDir(file) + return adder.addDir(path, file) } // case for symlink @@ -485,7 +485,7 @@ func (adder *Adder) addFile(file files.File) error { return err } - return adder.addNode(dagnode, s.FileName()) + return adder.addNode(dagnode, path) } // case for regular file @@ -493,7 +493,7 @@ func (adder *Adder) addFile(file files.File) error { // progress updates to the client (over the output channel) var reader io.Reader = file if adder.Progress { - rdr := &progressReader{file: file, out: adder.Out} + rdr := &progressReader{file: file, path: path, out: adder.Out} if fi, ok := file.(files.FileInfo); ok { reader = &progressReader2{rdr, fi} } else { @@ -506,26 +506,25 @@ func (adder *Adder) addFile(file files.File) error { return err } - addFileName := file.FileName() addFileInfo, ok := file.(files.FileInfo) if ok { if addFileInfo.AbsPath() == os.Stdin.Name() && adder.Name != "" { - addFileName = adder.Name + path = adder.Name adder.Name = "" } } // patch it into the root - return adder.addNode(dagnode, addFileName) + return adder.addNode(dagnode, path) } -func (adder *Adder) addDir(dir files.File) error { - log.Infof("adding directory: %s", dir.FileName()) +func (adder *Adder) addDir(path string, dir files.File) error { + log.Infof("adding directory: %s", path) mr, err := adder.mfsRoot() if err != nil { return err } - err = mfs.Mkdir(mr, dir.FileName(), mfs.MkdirOpts{ + err = mfs.Mkdir(mr, path, mfs.MkdirOpts{ Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, @@ -535,7 +534,7 @@ func (adder *Adder) addDir(dir files.File) error { } for { - file, err := dir.NextFile() + name, file, err := dir.NextFile() if err != nil && err != io.EOF { return err } @@ -543,12 +542,14 @@ func (adder *Adder) addDir(dir files.File) error { break } + fpath := gopath.Join(path, name) + // Skip hidden files when adding recursively, unless Hidden is enabled. - if files.IsHidden(file) && !adder.Hidden { - log.Infof("%s is hidden, skipping", file.FileName()) + if files.IsHidden(fpath, file) && !adder.Hidden { + log.Infof("%s is hidden, skipping", fpath) continue } - err = adder.addFile(file) + err = adder.addFile(fpath, file) if err != nil { return err } @@ -616,6 +617,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) { type progressReader struct { file files.File + path string out chan<- interface{} bytes int64 lastProgress int64 @@ -628,7 +630,7 @@ func (i *progressReader) Read(p []byte) (int, error) { if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF { i.lastProgress = i.bytes i.out <- &coreiface.AddEvent{ - Name: i.file.FileName(), + Name: i.path, Bytes: i.bytes, } } @@ -640,3 +642,7 @@ type progressReader2 struct { *progressReader files.FileInfo } + +func (i *progressReader2) Read(p []byte) (int, error) { + return i.progressReader.Read(p) +} diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index a96a63c1d..eb2e70a68 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -71,16 +71,20 @@ func TestAddGCLive(t *testing.T) { adder.Out = out dataa := ioutil.NopCloser(bytes.NewBufferString("testfileA")) - rfa := files.NewReaderFile("a", "a", dataa, nil) + rfa := files.NewReaderFile(dataa, nil) // make two files with pipes so we can 'pause' the add for timing of the test piper, pipew := io.Pipe() - hangfile := files.NewReaderFile("b", "b", piper, nil) + hangfile := files.NewReaderFile(piper, nil) datad := ioutil.NopCloser(bytes.NewBufferString("testfileD")) - rfd := files.NewReaderFile("d", "d", datad, nil) + rfd := files.NewReaderFile(datad, nil) - slf := files.NewSliceFile("files", "files", []files.File{rfa, hangfile, rfd}) + slf := files.NewSliceFile([]files.FileEntry{ + {File: rfa, Name: "a"}, + {File: hangfile, Name: "b"}, + {File: rfd, Name: "d"}, + }) addDone := make(chan struct{}) go func() { @@ -189,7 +193,7 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) { rand.New(rand.NewSource(2)).Read(data) // Rand.Read never returns an error fileData := ioutil.NopCloser(bytes.NewBuffer(data)) fileInfo := dummyFileInfo{"foo.txt", int64(len(data)), time.Now()} - file := files.NewReaderFile("foo.txt", "/tmp/foo.txt", fileData, &fileInfo) + file := files.NewReaderFile(fileData, &fileInfo) go func() { defer close(adder.Out)