Merge pull request #553 from jbenet/serial-file

Serial File Opening
This commit is contained in:
Juan Batiz-Benet 2015-01-13 23:30:08 -08:00
commit 920ddc7fc3
13 changed files with 339 additions and 260 deletions

View File

@ -5,12 +5,11 @@ import (
"errors"
"fmt"
"os"
fp "path"
"runtime"
"sort"
"strings"
cmds "github.com/jbenet/go-ipfs/commands"
files "github.com/jbenet/go-ipfs/commands/files"
u "github.com/jbenet/go-ipfs/util"
)
@ -65,7 +64,7 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
}
req.SetArguments(stringArgs)
file := &cmds.SliceFile{"", fileArgs}
file := &files.SliceFile{"", fileArgs}
req.SetFiles(file)
err = cmd.CheckArguments(req)
@ -139,7 +138,7 @@ func parseOptions(input []string) (map[string]interface{}, []string, error) {
return opts, args, nil
}
func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []cmds.File, error) {
func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []files.File, error) {
// ignore stdin on Windows
if runtime.GOOS == "windows" {
stdin = nil
@ -176,7 +175,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
}
stringArgs := make([]string, 0, numInputs)
fileArgs := make([]cmds.File, 0, numInputs)
fileArgs := make([]files.File, 0, numInputs)
argDefIndex := 0 // the index of the current argument definition
for i := 0; i < numInputs; i++ {
@ -264,7 +263,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
return append(args, strings.Split(input, "\n")...), nil, nil
}
func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]cmds.File, []string, error) {
func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) {
path := inputs[0]
file, err := os.Open(path)
@ -290,7 +289,7 @@ func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recurs
}
}
arg, err := openPath(file, path)
arg, err := files.NewSerialFile(path, file)
if err != nil {
return nil, nil, err
}
@ -298,51 +297,11 @@ func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recurs
return append(args, arg), inputs[1:], nil
}
func appendStdinAsFile(args []cmds.File, stdin *os.File) ([]cmds.File, *os.File) {
arg := &cmds.ReaderFile{"", stdin}
func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
arg := &files.ReaderFile{"", stdin}
return append(args, arg), nil
}
// recursively get file or directory contents as a cmds.File
func openPath(file *os.File, path string) (cmds.File, error) {
stat, err := file.Stat()
if err != nil {
return nil, err
}
// for non-directories, return a ReaderFile
if !stat.IsDir() {
return &cmds.ReaderFile{path, file}, nil
}
// for directories, recursively iterate though children then return as a SliceFile
contents, err := file.Readdir(0)
if err != nil {
return nil, err
}
// make sure contents are sorted so -- repeatably -- we get the same inputs.
sort.Sort(sortFIByName(contents))
files := make([]cmds.File, 0, len(contents))
for _, child := range contents {
childPath := fp.Join(path, child.Name())
childFile, err := os.Open(childPath)
if err != nil {
return nil, err
}
f, err := openPath(childFile, childPath)
if err != nil {
return nil, err
}
files = append(files, f)
}
return &cmds.SliceFile{path, files}, nil
}
// isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),
// and false otherwise (e.g. nothing is being piped in, so stdin is
// coming from the terminal)
@ -355,9 +314,3 @@ func isTerminal(stdin *os.File) (bool, error) {
// if stdin is a CharDevice, return true
return ((stat.Mode() & os.ModeCharDevice) != 0), nil
}
type sortFIByName []os.FileInfo
func (es sortFIByName) Len() int { return len(es) }
func (es sortFIByName) Swap(i, j int) { es[i], es[j] = es[j], es[i] }
func (es sortFIByName) Less(i, j int) bool { return es[i].Name() < es[j].Name() }

View File

@ -1,176 +0,0 @@
package commands
import (
"errors"
"io"
"mime"
"mime/multipart"
"net/http"
)
const (
multipartFormdataType = "multipart/form-data"
multipartMixedType = "multipart/mixed"
contentTypeHeader = "Content-Type"
)
var (
ErrNotDirectory = errors.New("Couln't call NextFile(), this isn't a directory")
ErrNotReader = errors.New("This file is a directory, can't use Reader functions")
)
// File is an interface that provides functionality for handling files/directories
// as values that can be supplied to commands. For directories, child files are
// accessed serially by calling `NextFile()`.
type File interface {
// Files implement ReadCloser, but can only be read from or closed if they are not directories
io.ReadCloser
// FileName returns a full filename path associated with this file
FileName() string
// IsDirectory returns true if the File is a directory (and therefore supports calling `NextFile`)
// and false if the File is a normal file (and therefor supports calling `Read` and `Close`)
IsDirectory() bool
// NextFile returns the next child file available (if the File is a directory).
// It will return (nil, io.EOF) if no more files are available.
// If the file is a regular file (not a directory), NextFile will return a non-nil error.
NextFile() (File, error)
}
// MultipartFile implements File, and is created from a `multipart.Part`.
// It can be either a directory or file (checked by calling `IsDirectory()`).
type MultipartFile struct {
File
Part *multipart.Part
Reader *multipart.Reader
Mediatype string
}
func NewFileFromPart(part *multipart.Part) (File, error) {
f := &MultipartFile{
Part: part,
}
contentType := part.Header.Get(contentTypeHeader)
var params map[string]string
var err error
f.Mediatype, params, err = mime.ParseMediaType(contentType)
if err != nil {
return nil, err
}
if f.IsDirectory() {
boundary, found := params["boundary"]
if !found {
return nil, http.ErrMissingBoundary
}
f.Reader = multipart.NewReader(part, boundary)
}
return f, nil
}
func (f *MultipartFile) IsDirectory() bool {
return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
}
func (f *MultipartFile) NextFile() (File, error) {
if !f.IsDirectory() {
return nil, ErrNotDirectory
}
part, err := f.Reader.NextPart()
if err != nil {
return nil, err
}
return NewFileFromPart(part)
}
func (f *MultipartFile) FileName() string {
return f.Part.FileName()
}
func (f *MultipartFile) Read(p []byte) (int, error) {
if f.IsDirectory() {
return 0, ErrNotReader
}
return f.Part.Read(p)
}
func (f *MultipartFile) Close() error {
if f.IsDirectory() {
return ErrNotReader
}
return f.Part.Close()
}
// SliceFile implements File, and provides simple directory handling.
// It contains children files, and is created from a `[]File`.
// SliceFiles are always directories, and can't be read from or closed.
type SliceFile struct {
Filename string
Files []File
}
func (f *SliceFile) IsDirectory() bool {
return true
}
func (f *SliceFile) NextFile() (File, error) {
if len(f.Files) == 0 {
return nil, io.EOF
}
file := f.Files[0]
f.Files = f.Files[1:]
return file, nil
}
func (f *SliceFile) FileName() string {
return f.Filename
}
func (f *SliceFile) Read(p []byte) (int, error) {
return 0, ErrNotReader
}
func (f *SliceFile) Close() error {
return ErrNotReader
}
// ReaderFile is a implementation of File created from an `io.Reader`.
// ReaderFiles are never directories, and can be read from and closed.
type ReaderFile struct {
Filename string
Reader io.Reader
}
func (f *ReaderFile) IsDirectory() bool {
return false
}
func (f *ReaderFile) NextFile() (File, error) {
return nil, ErrNotDirectory
}
func (f *ReaderFile) FileName() string {
return f.Filename
}
func (f *ReaderFile) Read(p []byte) (int, error) {
return f.Reader.Read(p)
}
func (f *ReaderFile) Close() error {
closer, ok := f.Reader.(io.Closer)
if !ok {
return nil
}
return closer.Close()
}

31
commands/files/file.go Normal file
View File

@ -0,0 +1,31 @@
package files
import (
"errors"
"io"
)
var (
ErrNotDirectory = errors.New("Couln't call NextFile(), this isn't a directory")
ErrNotReader = errors.New("This file is a directory, can't use Reader functions")
)
// File is an interface that provides functionality for handling files/directories
// as values that can be supplied to commands. For directories, child files are
// accessed serially by calling `NextFile()`.
type File interface {
// Files implement ReadCloser, but can only be read from or closed if they are not directories
io.ReadCloser
// FileName returns a full filename path associated with this file
FileName() string
// IsDirectory returns true if the File is a directory (and therefore supports calling `NextFile`)
// and false if the File is a normal file (and therefor supports calling `Read` and `Close`)
IsDirectory() bool
// NextFile returns the next child file available (if the File is a directory).
// It will return (nil, io.EOF) if no more files are available.
// If the file is a regular file (not a directory), NextFile will return a non-nil error.
NextFile() (File, error)
}

View File

@ -1,7 +1,8 @@
package commands
package files
import (
"io"
"io/ioutil"
"mime/multipart"
"strings"
"testing"
@ -10,9 +11,9 @@ import (
func TestSliceFiles(t *testing.T) {
name := "testname"
files := []File{
&ReaderFile{"file.txt", strings.NewReader("Some text!\n")},
&ReaderFile{"beep.txt", strings.NewReader("beep")},
&ReaderFile{"boop.txt", strings.NewReader("boop")},
&ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n"))},
&ReaderFile{"beep.txt", ioutil.NopCloser(strings.NewReader("beep"))},
&ReaderFile{"boop.txt", ioutil.NopCloser(strings.NewReader("boop"))},
}
buf := make([]byte, 20)
@ -54,7 +55,7 @@ func TestSliceFiles(t *testing.T) {
func TestReaderFiles(t *testing.T) {
message := "beep boop"
rf := &ReaderFile{"file.txt", strings.NewReader(message)}
rf := &ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader(message))}
buf := make([]byte, len(message))
if rf.IsDirectory() {

View File

@ -0,0 +1,85 @@
package files
import (
"mime"
"mime/multipart"
"net/http"
)
const (
multipartFormdataType = "multipart/form-data"
multipartMixedType = "multipart/mixed"
contentTypeHeader = "Content-Type"
)
// MultipartFile implements File, and is created from a `multipart.Part`.
// It can be either a directory or file (checked by calling `IsDirectory()`).
type MultipartFile struct {
File
Part *multipart.Part
Reader *multipart.Reader
Mediatype string
}
func NewFileFromPart(part *multipart.Part) (File, error) {
f := &MultipartFile{
Part: part,
}
contentType := part.Header.Get(contentTypeHeader)
var params map[string]string
var err error
f.Mediatype, params, err = mime.ParseMediaType(contentType)
if err != nil {
return nil, err
}
if f.IsDirectory() {
boundary, found := params["boundary"]
if !found {
return nil, http.ErrMissingBoundary
}
f.Reader = multipart.NewReader(part, boundary)
}
return f, nil
}
func (f *MultipartFile) IsDirectory() bool {
return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
}
func (f *MultipartFile) NextFile() (File, error) {
if !f.IsDirectory() {
return nil, ErrNotDirectory
}
part, err := f.Reader.NextPart()
if err != nil {
return nil, err
}
return NewFileFromPart(part)
}
func (f *MultipartFile) FileName() string {
return f.Part.FileName()
}
func (f *MultipartFile) Read(p []byte) (int, error) {
if f.IsDirectory() {
return 0, ErrNotReader
}
return f.Part.Read(p)
}
func (f *MultipartFile) Close() error {
if f.IsDirectory() {
return ErrNotReader
}
return f.Part.Close()
}

View File

@ -0,0 +1,30 @@
package files
import "io"
// ReaderFile is a implementation of File created from an `io.Reader`.
// ReaderFiles are never directories, and can be read from and closed.
type ReaderFile struct {
Filename string
Reader io.ReadCloser
}
func (f *ReaderFile) IsDirectory() bool {
return false
}
func (f *ReaderFile) NextFile() (File, error) {
return nil, ErrNotDirectory
}
func (f *ReaderFile) FileName() string {
return f.Filename
}
func (f *ReaderFile) Read(p []byte) (int, error) {
return f.Reader.Read(p)
}
func (f *ReaderFile) Close() error {
return f.Reader.Close()
}

View File

@ -0,0 +1,115 @@
package files
import (
"io"
"os"
fp "path"
"sort"
"syscall"
)
type sortFIByName []os.FileInfo
func (es sortFIByName) Len() int { return len(es) }
func (es sortFIByName) Swap(i, j int) { es[i], es[j] = es[j], es[i] }
func (es sortFIByName) Less(i, j int) bool { return es[i].Name() < es[j].Name() }
// serialFile implements File, and reads from a path on the OS filesystem.
// No more than one file will be opened at a time (directories will advance
// to the next file when NextFile() is called).
type serialFile struct {
path string
files []os.FileInfo
current *os.File
}
func NewSerialFile(path string, file *os.File) (File, error) {
stat, err := file.Stat()
if err != nil {
return nil, err
}
return newSerialFile(path, file, stat)
}
func newSerialFile(path string, file *os.File, stat os.FileInfo) (File, error) {
// for non-directories, return a ReaderFile
if !stat.IsDir() {
return &ReaderFile{path, file}, nil
}
// for directories, stat all of the contents first, so we know what files to
// open when NextFile() is called
contents, err := file.Readdir(0)
if err != nil {
return nil, err
}
// we no longer need our root directory file (we already statted the contents),
// so close it
err = file.Close()
if err != nil {
return nil, err
}
// make sure contents are sorted so -- repeatably -- we get the same inputs.
sort.Sort(sortFIByName(contents))
return &serialFile{path, contents, nil}, nil
}
func (f *serialFile) IsDirectory() bool {
// non-directories get created as a ReaderFile, so serialFiles should only
// represent directories
return true
}
func (f *serialFile) NextFile() (File, error) {
// if a file was opened previously, close it
err := f.Close()
if err != nil {
return nil, err
}
// if there aren't any files left in the root directory, we're done
if len(f.files) == 0 {
return nil, io.EOF
}
stat := f.files[0]
f.files = f.files[1:]
// open the next file
filePath := fp.Join(f.path, stat.Name())
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
f.current = file
// recursively call the constructor on the next file
// if it's a regular file, we will open it as a ReaderFile
// if it's a directory, files in it will be opened serially
return newSerialFile(filePath, file, stat)
}
func (f *serialFile) FileName() string {
return f.path
}
func (f *serialFile) Read(p []byte) (int, error) {
return 0, ErrNotReader
}
func (f *serialFile) Close() error {
// close the current file if there is one
if f.current != nil {
err := f.current.Close()
// ignore EINVAL error, the file might have already been closed
if err != nil && err != syscall.EINVAL {
return err
}
}
return nil
}

View File

@ -0,0 +1,36 @@
package files
import "io"
// SliceFile implements File, and provides simple directory handling.
// It contains children files, and is created from a `[]File`.
// SliceFiles are always directories, and can't be read from or closed.
type SliceFile struct {
Filename string
Files []File
}
func (f *SliceFile) IsDirectory() bool {
return true
}
func (f *SliceFile) NextFile() (File, error) {
if len(f.Files) == 0 {
return nil, io.EOF
}
file := f.Files[0]
f.Files = f.Files[1:]
return file, nil
}
func (f *SliceFile) FileName() string {
return f.Filename
}
func (f *SliceFile) Read(p []byte) (int, error) {
return 0, ErrNotReader
}
func (f *SliceFile) Close() error {
return ErrNotReader
}

View File

@ -8,7 +8,7 @@ import (
"net/textproto"
"sync"
cmds "github.com/jbenet/go-ipfs/commands"
files "github.com/jbenet/go-ipfs/commands/files"
)
// MultiFileReader reads from a `commands.File` (which can be a directory of files
@ -16,7 +16,7 @@ import (
type MultiFileReader struct {
io.Reader
files cmds.File
files files.File
currentFile io.Reader
buf bytes.Buffer
mpWriter *multipart.Writer
@ -31,7 +31,7 @@ type MultiFileReader struct {
// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`.
// If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data',
// if `form` is false, the Content-Type will be 'multipart/mixed'.
func NewMultiFileReader(file cmds.File, form bool) *MultiFileReader {
func NewMultiFileReader(file files.File, form bool) *MultiFileReader {
mfr := &MultiFileReader{
files: file,
form: form,

View File

@ -2,24 +2,25 @@ package http
import (
"io"
"io/ioutil"
"mime/multipart"
"strings"
"testing"
cmds "github.com/jbenet/go-ipfs/commands"
files "github.com/jbenet/go-ipfs/commands/files"
)
func TestOutput(t *testing.T) {
text := "Some text! :)"
files := []cmds.File{
&cmds.ReaderFile{"file.txt", strings.NewReader(text)},
&cmds.SliceFile{"boop", []cmds.File{
&cmds.ReaderFile{"boop/a.txt", strings.NewReader("bleep")},
&cmds.ReaderFile{"boop/b.txt", strings.NewReader("bloop")},
fileset := []files.File{
&files.ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader(text))},
&files.SliceFile{"boop", []files.File{
&files.ReaderFile{"boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep"))},
&files.ReaderFile{"boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop"))},
}},
&cmds.ReaderFile{"beep.txt", strings.NewReader("beep")},
&files.ReaderFile{"beep.txt", ioutil.NopCloser(strings.NewReader("beep"))},
}
sf := &cmds.SliceFile{"", files}
sf := &files.SliceFile{"", fileset}
buf := make([]byte, 20)
// testing output by reading it with the go stdlib "mime/multipart" Reader
@ -30,7 +31,7 @@ func TestOutput(t *testing.T) {
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err := cmds.NewFileFromPart(part)
mpf, err := files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}
@ -51,7 +52,7 @@ func TestOutput(t *testing.T) {
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err = cmds.NewFileFromPart(part)
mpf, err = files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}
@ -93,7 +94,7 @@ func TestOutput(t *testing.T) {
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err = cmds.NewFileFromPart(part)
mpf, err = files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}

View File

@ -8,6 +8,7 @@ import (
"strings"
cmds "github.com/jbenet/go-ipfs/commands"
files "github.com/jbenet/go-ipfs/commands/files"
)
// Parse parses the data in a http.Request and returns a command Request object
@ -94,9 +95,9 @@ func Parse(r *http.Request, root *cmds.Command) (cmds.Request, error) {
contentType := r.Header.Get(contentTypeHeader)
mediatype, _, _ := mime.ParseMediaType(contentType)
var f *cmds.MultipartFile
var f *files.MultipartFile
if mediatype == "multipart/form-data" {
f = &cmds.MultipartFile{Mediatype: mediatype}
f = &files.MultipartFile{Mediatype: mediatype}
f.Reader, err = r.MultipartReader()
if err != nil {
return nil, err

View File

@ -8,6 +8,7 @@ import (
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
"github.com/jbenet/go-ipfs/commands/files"
"github.com/jbenet/go-ipfs/core"
"github.com/jbenet/go-ipfs/repo/config"
u "github.com/jbenet/go-ipfs/util"
@ -71,8 +72,8 @@ type Request interface {
SetOptions(opts map[string]interface{}) error
Arguments() []string
SetArguments([]string)
Files() File
SetFiles(File)
Files() files.File
SetFiles(files.File)
Context() *Context
SetContext(Context)
Command() *Command
@ -84,7 +85,7 @@ type request struct {
path []string
options optMap
arguments []string
files File
files files.File
cmd *Command
ctx Context
optionDefs map[string]Option
@ -159,11 +160,11 @@ func (r *request) SetArguments(args []string) {
r.arguments = args
}
func (r *request) Files() File {
func (r *request) Files() files.File {
return r.files
}
func (r *request) SetFiles(f File) {
func (r *request) SetFiles(f files.File) {
r.files = f
}
@ -259,7 +260,7 @@ func NewEmptyRequest() (Request, error) {
// NewRequest returns a request initialized with given arguments
// An non-nil error will be returned if the provided option values are invalid
func NewRequest(path []string, opts optMap, args []string, file File, cmd *Command, optDefs map[string]Option) (Request, error) {
func NewRequest(path []string, opts optMap, args []string, file files.File, cmd *Command, optDefs map[string]Option) (Request, error) {
if path == nil {
path = make([]string, 0)
}

View File

@ -8,6 +8,7 @@ import (
"path"
cmds "github.com/jbenet/go-ipfs/commands"
files "github.com/jbenet/go-ipfs/commands/files"
core "github.com/jbenet/go-ipfs/core"
importer "github.com/jbenet/go-ipfs/importer"
"github.com/jbenet/go-ipfs/importer/chunk"
@ -138,7 +139,7 @@ func addNode(n *core.IpfsNode, node *dag.Node) error {
return nil
}
func addFile(n *core.IpfsNode, file cmds.File, out chan interface{}) (*dag.Node, error) {
func addFile(n *core.IpfsNode, file files.File, out chan interface{}) (*dag.Node, error) {
if file.IsDirectory() {
return addDir(n, file, out)
}
@ -155,7 +156,7 @@ func addFile(n *core.IpfsNode, file cmds.File, out chan interface{}) (*dag.Node,
return dns[len(dns)-1], nil // last dag node is the file.
}
func addDir(n *core.IpfsNode, dir cmds.File, out chan interface{}) (*dag.Node, error) {
func addDir(n *core.IpfsNode, dir files.File, out chan interface{}) (*dag.Node, error) {
log.Infof("adding directory: %s", dir.FileName())
tree := &dag.Node{Data: ft.FolderPBData()}