diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 92168503e..69e731822 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -13,8 +13,16 @@ import ( // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { // Add imports the data from the reader into merkledag file + // + // TODO: a long useful comment on how to use this for many different scenarios Add(context.Context, files.File, ...options.UnixfsAddOption) (ResolvedPath, error) + // Get returns a read-only handle to a file tree referenced by a path + // + // Note that some implementations of this API may apply the specified context + // to operations performed on the returned file + Get(context.Context, Path) (files.File, error) + // Cat returns a reader for the file Cat(context.Context, Path) (Reader, error) diff --git a/core/coreapi/unixfile.go b/core/coreapi/unixfile.go new file mode 100644 index 000000000..953350754 --- /dev/null +++ b/core/coreapi/unixfile.go @@ -0,0 +1,199 @@ +package coreapi + +import ( + "bytes" + "context" + "errors" + "io" + "io/ioutil" + "os" + gopath "path" + "time" + + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" + uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" + dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" + ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" +) + +// Number to file to prefetch in directories +// TODO: should we allow setting this via context hint? +const prefetchFiles = 4 + +// TODO: this probably belongs in go-unixfs (and could probably replace a chunk of it's interface in the long run) + +type sizeInfo struct { + size int64 + name string + modTime time.Time +} + +func (s *sizeInfo) Name() string { + return s.name +} + +func (s *sizeInfo) Size() int64 { + return s.size +} + +func (s *sizeInfo) Mode() os.FileMode { + return 0444 // all read +} + +func (s *sizeInfo) ModTime() time.Time { + return s.modTime +} + +func (s *sizeInfo) IsDir() bool { + return false +} + +func (s *sizeInfo) Sys() interface{} { + return nil +} + +type ufsDirectory struct { + ctx context.Context + dserv ipld.DAGService + + files chan *ipld.Link + + name string + path string +} + +func (d *ufsDirectory) Close() error { + return files.ErrNotReader +} + +func (d *ufsDirectory) Read(_ []byte) (int, error) { + return 0, files.ErrNotReader +} + +func (d *ufsDirectory) FileName() string { + return d.name +} + +func (d *ufsDirectory) FullPath() string { + return d.path +} + +func (d *ufsDirectory) IsDirectory() bool { + return true +} + +func (d *ufsDirectory) NextFile() (files.File, error) { + l, ok := <-d.files + if !ok { + return nil, io.EOF + } + + nd, err := l.GetNode(d.ctx, d.dserv) + if err != nil { + return nil, err + } + + return newUnixfsFile(d.ctx, d.dserv, nd, l.Name, d) +} + +type ufsFile struct { + uio.DagReader + + name string + path string +} + +func (f *ufsFile) IsDirectory() bool { + return false +} + +func (f *ufsFile) NextFile() (files.File, error) { + return nil, files.ErrNotDirectory +} + +func (f *ufsFile) FileName() string { + return f.name +} + +func (f *ufsFile) FullPath() string { + return f.path +} + +func (f *ufsFile) Size() (int64, error) { + return int64(f.DagReader.Size()), nil +} + +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, path string) (files.File, error) { + dir, err := uio.NewDirectoryFromNode(dserv, nd) + if err != nil { + return nil, err + } + + fileCh := make(chan *ipld.Link, prefetchFiles) + go func() { + dir.ForEachLink(ctx, func(link *ipld.Link) error { + select { + case fileCh <- link: + case <-ctx.Done(): + return ctx.Err() + } + return nil + }) + + close(fileCh) + }() + + return &ufsDirectory{ + ctx: ctx, + dserv: dserv, + + files: fileCh, + + name: name, + path: path, + }, nil +} + +func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, parent files.File) (files.File, error) { + path := name + if parent != nil { + path = gopath.Join(parent.FullPath(), name) + } + + switch dn := nd.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.RawData()) + if err != nil { + return nil, err + } + if fsn.IsDir() { + return newUnixfsDir(ctx, dserv, nd, name, path) + } + + case *dag.RawNode: + + r := ioutil.NopCloser(bytes.NewReader(dn.RawData())) + fi := &sizeInfo{ + size: int64(len(dn.RawData())), + } + + return files.NewReaderFile("", "", r, fi), nil + default: + return nil, errors.New("unknown node type") + } + + dr, err := uio.NewDagReader(ctx, nd, dserv) + if err != nil { + return nil, err + } + + return &ufsFile{ + DagReader: dr, + + name: name, + path: path, + }, nil +} + +var _ os.FileInfo = &sizeInfo{} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 9ba1c29c3..80f59d278 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -110,6 +110,15 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options return coreiface.IpfsPath(nd.Cid()), nil } +func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.File, error) { + nd, err := api.core().ResolveNode(ctx, p) + if err != nil { + return nil, err + } + + return newUnixfsFile(ctx, api.node.DAG, nd, "", nil) +} + // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. func (api *UnixfsAPI) Cat(ctx context.Context, p coreiface.Path) (coreiface.Reader, error) { dget := api.node.DAG // TODO: use a session here once routing perf issues are resolved