From c5a69db601bf291fc42b1a92f04377bd9eadb88e Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 23 Mar 2015 09:51:44 +0100 Subject: [PATCH] updated godeps --- Godeps/Godeps.json | 32 +- .../src/bazil.org/fuse/error_darwin.go | 39 +- .../src/bazil.org/fuse/error_freebsd.go | 15 + .../src/bazil.org/fuse/error_linux.go | 17 + .../src/bazil.org/fuse/error_std.go | 34 +- .../bazil.org/fuse/fs/fstestutil/mountinfo.go | 12 + .../fuse/fs/fstestutil/mountinfo_darwin.go | 12 - .../fuse/fs/fstestutil/mountinfo_freebsd.go | 7 + .../fuse/fs/fstestutil/record/record.go | 7 +- .../bazil.org/fuse/fs/fstestutil/testfs.go | 19 + .../src/bazil.org/fuse/fs/helpers_test.go | 67 ++ .../_workspace/src/bazil.org/fuse/fs/serve.go | 36 +- .../src/bazil.org/fuse/fs/serve_test.go | 255 +++-- Godeps/_workspace/src/bazil.org/fuse/fuse.go | 61 +- .../src/bazil.org/fuse/fuse_kernel_freebsd.go | 60 ++ .../src/bazil.org/fuse/mount_darwin.go | 4 +- .../src/bazil.org/fuse/mount_freebsd.go | 41 + .../_workspace/src/bazil.org/fuse/options.go | 32 + .../src/bazil.org/fuse/options_darwin_test.go | 27 - .../src/bazil.org/fuse/options_freebsd.go | 9 + .../src/bazil.org/fuse/options_helper_test.go | 6 + .../src/bazil.org/fuse/options_linux.go | 4 - .../bazil.org/fuse/options_nocomma_test.go | 34 + .../src/bazil.org/fuse/options_test.go | 85 ++ .../bazil.org/fuse/syscallx/syscallx_std.go | 12 +- .../src/github.com/Sirupsen/logrus/README.md | 51 +- .../Sirupsen/logrus/examples/basic/basic.go | 10 + .../Sirupsen/logrus/examples/hook/hook.go | 10 +- .../logrus/formatters/logstash/logstash.go | 48 + .../formatters/logstash/logstash_test.go | 52 + .../logrus/hooks/airbrake/airbrake.go | 58 +- .../logrus/hooks/airbrake/airbrake_test.go | 133 +++ .../Sirupsen/logrus/hooks/bugsnag/bugsnag.go | 68 ++ .../logrus/hooks/bugsnag/bugsnag_test.go | 64 ++ .../Sirupsen/logrus/hooks/sentry/README.md | 2 +- .../Sirupsen/logrus/json_formatter.go | 9 +- .../Sirupsen/logrus/json_formatter_test.go | 120 +++ .../src/github.com/Sirupsen/logrus/logger.go | 84 +- .../Sirupsen/logrus/terminal_openbsd.go | 1 - .../Sirupsen/logrus/text_formatter.go | 18 +- .../Sirupsen/logrus/text_formatter_test.go | 4 + .../src/github.com/Sirupsen/logrus/writer.go | 2 +- .../coreos/go-semver/semver/semver.go | 7 + .../coreos/go-semver/semver/semver_test.go | 36 + .../github.com/gorilla/context/.travis.yml | 7 - .../src/github.com/gorilla/context/LICENSE | 27 - .../src/github.com/gorilla/context/README.md | 7 - .../src/github.com/gorilla/context/context.go | 143 --- .../gorilla/context/context_test.go | 161 --- .../src/github.com/gorilla/context/doc.go | 82 -- .../src/github.com/gorilla/mux/.travis.yml | 7 - .../src/github.com/gorilla/mux/LICENSE | 27 - .../src/github.com/gorilla/mux/README.md | 7 - .../src/github.com/gorilla/mux/bench_test.go | 21 - .../src/github.com/gorilla/mux/doc.go | 199 ---- .../src/github.com/gorilla/mux/mux.go | 353 ------- .../src/github.com/gorilla/mux/mux_test.go | 943 ------------------ .../src/github.com/gorilla/mux/old_test.go | 714 ------------- .../src/github.com/gorilla/mux/regexp.go | 274 ----- .../src/github.com/gorilla/mux/route.go | 524 ---------- .../jbenet/go-base58/base58_test.go | 34 + .../jbenet/go-reuseport/available_unix.go | 1 + .../jbenet/go-reuseport/impl_windows.go | 12 +- .../jbenet/go-reuseport/opts_posix.go | 2 +- .../jbenet/go-reuseport/poll/poll_bsd.go | 2 +- .../go-reuseport/poll/poll_darwin_386.go | 57 ++ .../kardianos/osext/osext_procfs.go | 8 +- .../github.com/kardianos/osext/osext_test.go | 137 ++- .../{go13_bench_test.go => bench2_test.go} | 2 +- .../goleveldb/leveldb/cache/bench2_test.go | 30 + .../syndtr/goleveldb/leveldb/cache/cache.go | 753 +++++++++++--- .../goleveldb/leveldb/cache/cache_test.go | 883 ++++++++-------- .../syndtr/goleveldb/leveldb/cache/lru.go | 195 ++++ .../goleveldb/leveldb/cache/lru_cache.go | 622 ------------ .../syndtr/goleveldb/leveldb/corrupt_test.go | 18 +- .../github.com/syndtr/goleveldb/leveldb/db.go | 21 +- .../syndtr/goleveldb/leveldb/db_test.go | 14 +- .../syndtr/goleveldb/leveldb/external_test.go | 16 +- .../syndtr/goleveldb/leveldb/opt/options.go | 133 ++- .../syndtr/goleveldb/leveldb/options.go | 8 - .../syndtr/goleveldb/leveldb/session.go | 5 +- .../syndtr/goleveldb/leveldb/session_util.go | 4 +- .../syndtr/goleveldb/leveldb/table.go | 70 +- .../syndtr/goleveldb/leveldb/table/reader.go | 63 +- .../syndtr/goleveldb/leveldb/table/writer.go | 3 +- .../syndtr/gosnappy/snappy/decode.go | 172 +++- .../syndtr/gosnappy/snappy/encode.go | 84 ++ .../syndtr/gosnappy/snappy/snappy.go | 30 + .../syndtr/gosnappy/snappy/snappy_test.go | 201 +++- 89 files changed, 3437 insertions(+), 5343 deletions(-) create mode 100644 Godeps/_workspace/src/bazil.org/fuse/error_freebsd.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/error_linux.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_freebsd.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/fs/helpers_test.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/fuse_kernel_freebsd.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/mount_freebsd.go delete mode 100644 Godeps/_workspace/src/bazil.org/fuse/options_darwin_test.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/options_freebsd.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/options_helper_test.go create mode 100644 Godeps/_workspace/src/bazil.org/fuse/options_nocomma_test.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash_test.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake_test.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag_test.go create mode 100644 Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter_test.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/.travis.yml delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/LICENSE delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/README.md delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/context.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/context_test.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/context/doc.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/.travis.yml delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/LICENSE delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/README.md delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/bench_test.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/doc.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/mux.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/mux_test.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/old_test.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/regexp.go delete mode 100644 Godeps/_workspace/src/github.com/gorilla/mux/route.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_darwin_386.go rename Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/{go13_bench_test.go => bench2_test.go} (98%) create mode 100644 Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go create mode 100644 Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go delete mode 100644 Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 6cfab9ecb..05c837213 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -7,7 +7,7 @@ "Deps": [ { "ImportPath": "bazil.org/fuse", - "Rev": "489e985dcb429718328be2bb055adb685c4ae1e3" + "Rev": "79d103f5608724e3ccee0a10daccc5e4aff03591" }, { "ImportPath": "code.google.com/p/go-uuid/uuid", @@ -33,8 +33,8 @@ }, { "ImportPath": "github.com/Sirupsen/logrus", - "Comment": "v0.6.5", - "Rev": "c0f7e35ed2e48f188c37581b4b743cf7383f85c6" + "Comment": "v0.7.1", + "Rev": "3fc34d061b9c78a70db853c7cb6b0576b6d4f32d" }, { "ImportPath": "github.com/braintree/manners", @@ -67,7 +67,7 @@ }, { "ImportPath": "github.com/coreos/go-semver/semver", - "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa" + "Rev": "568e959cd89871e61434c1143528d9162da89ef2" }, { "ImportPath": "github.com/crowdmob/goamz/aws", @@ -102,14 +102,6 @@ "Comment": "v0.5.1", "Rev": "27a863cdffdb0998d13e1e11992b18489aeeaa25" }, - { - "ImportPath": "github.com/gorilla/context", - "Rev": "14f550f51af52180c2eefed15e5fd18d63c0a64a" - }, - { - "ImportPath": "github.com/gorilla/mux", - "Rev": "4b8fbc56f3b2400a7c7ea3dba9b3539787c486b6" - }, { "ImportPath": "github.com/h2so5/utp", "Rev": "654d875bb65e96729678180215cf080fe2810371" @@ -141,7 +133,7 @@ }, { "ImportPath": "github.com/jbenet/go-base58", - "Rev": "568a28d73fd97651d3442392036a658b6976eed5" + "Rev": "6237cf65f3a6f7111cd8a42be3590df99a66bc7d" }, { "ImportPath": "github.com/jbenet/go-ctxgroup", @@ -195,7 +187,7 @@ }, { "ImportPath": "github.com/jbenet/go-reuseport", - "Rev": "096958438ae3683c9f3c8ae0f7139b5ce600e6a8" + "Rev": "c71a70ef82a7acb87ad77704de1be7b9cd776a55" }, { "ImportPath": "github.com/jbenet/go-sockaddr/net", @@ -211,7 +203,7 @@ }, { "ImportPath": "github.com/kardianos/osext", - "Rev": "ccfcd0245381f0c94c68f50626665eed3c6b726a" + "Rev": "efacde03154693404c65e7aa7d461ac9014acd0c" }, { "ImportPath": "github.com/kr/binarydist", @@ -223,23 +215,23 @@ }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", - "Rev": "871eee0a7546bb7d1b2795142e29c4534abc49b3" + "Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e" }, { "ImportPath": "github.com/syndtr/gosnappy/snappy", - "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862" + "Rev": "156a073208e131d7d2e212cb749feae7c339e846" }, { "ImportPath": "golang.org/x/crypto/blowfish", - "Rev": "1351f936d976c60a0a48d728281922cf63eafb8d" + "Rev": "b7d6bf2c61544745a02f83dec90393985fc3a065" }, { "ImportPath": "golang.org/x/crypto/sha3", - "Rev": "1351f936d976c60a0a48d728281922cf63eafb8d" + "Rev": "b7d6bf2c61544745a02f83dec90393985fc3a065" }, { "ImportPath": "golang.org/x/net/context", - "Rev": "b6fdb7d8a4ccefede406f8fe0f017fb58265054c" + "Rev": "7dbad50ab5b31073856416cdcfeb2796d682f844" }, { "ImportPath": "gopkg.in/fsnotify.v1", diff --git a/Godeps/_workspace/src/bazil.org/fuse/error_darwin.go b/Godeps/_workspace/src/bazil.org/fuse/error_darwin.go index 195b0a772..a3fb89ca2 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/error_darwin.go +++ b/Godeps/_workspace/src/bazil.org/fuse/error_darwin.go @@ -4,33 +4,14 @@ import ( "syscall" ) -type getxattrError struct { - error -} - -func (getxattrError) Errno() Errno { - return Errno(syscall.ENOATTR) -} - -// getxattr return value for "extended attribute does not exist" is -// ENOATTR on OS X, and ENODATA on Linux and apparently at least -// NetBSD. There may be a #define ENOATTR too, but the value is -// ENODATA in the actual syscalls. ENOATTR is not in any of the -// standards, ENODATA exists but is only used for STREAMs. -// -// https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/getxattr.2.html -// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013090.html -// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013097.html -// http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html -func translateGetxattrError(err error) error { - ferr, ok := err.(ErrorNumber) - if !ok { - return err - } - - if ferr.Errno() != ENODATA { - return err - } - - return getxattrError{err} +const ( + ENOATTR = Errno(syscall.ENOATTR) +) + +const ( + errNoXattr = ENOATTR +) + +func init() { + errnoNames[errNoXattr] = "ENOATTR" } diff --git a/Godeps/_workspace/src/bazil.org/fuse/error_freebsd.go b/Godeps/_workspace/src/bazil.org/fuse/error_freebsd.go new file mode 100644 index 000000000..c6ea6d6e7 --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/error_freebsd.go @@ -0,0 +1,15 @@ +package fuse + +import "syscall" + +const ( + ENOATTR = Errno(syscall.ENOATTR) +) + +const ( + errNoXattr = ENOATTR +) + +func init() { + errnoNames[errNoXattr] = "ENOATTR" +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/error_linux.go b/Godeps/_workspace/src/bazil.org/fuse/error_linux.go new file mode 100644 index 000000000..6f113e71e --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/error_linux.go @@ -0,0 +1,17 @@ +package fuse + +import ( + "syscall" +) + +const ( + ENODATA = Errno(syscall.ENODATA) +) + +const ( + errNoXattr = ENODATA +) + +func init() { + errnoNames[errNoXattr] = "ENODATA" +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/error_std.go b/Godeps/_workspace/src/bazil.org/fuse/error_std.go index 76aaaf574..398f43fbf 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/error_std.go +++ b/Godeps/_workspace/src/bazil.org/fuse/error_std.go @@ -1,7 +1,31 @@ -// +build !darwin - package fuse -func translateGetxattrError(err error) error { - return err -} +// There is very little commonality in extended attribute errors +// across platforms. +// +// getxattr return value for "extended attribute does not exist" is +// ENOATTR on OS X, and ENODATA on Linux and apparently at least +// NetBSD. There may be a #define ENOATTR on Linux too, but the value +// is ENODATA in the actual syscalls. FreeBSD and OpenBSD have no +// ENODATA, only ENOATTR. ENOATTR is not in any of the standards, +// ENODATA exists but is only used for STREAMs. +// +// Each platform will define it a errNoXattr constant, and this file +// will enforce that it implements the right interfaces and hide the +// implementation. +// +// https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/getxattr.2.html +// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013090.html +// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013097.html +// http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html +// http://www.freebsd.org/cgi/man.cgi?query=extattr_get_file&sektion=2 +// http://nixdoc.net/man-pages/openbsd/man2/extattr_get_file.2.html + +// ErrNoXattr is a platform-independent error value meaning the +// extended attribute was not found. It can be used to respond to +// GetxattrRequest and such. +const ErrNoXattr = errNoXattr + +var _ error = ErrNoXattr +var _ Errno = ErrNoXattr +var _ ErrorNumber = ErrNoXattr diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo.go b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo.go index 4e410ebd0..654417bc4 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo.go @@ -12,3 +12,15 @@ type MountInfo struct { func GetMountInfo(mnt string) (*MountInfo, error) { return getMountInfo(mnt) } + +// cstr converts a nil-terminated C string into a Go string +func cstr(ca []int8) string { + s := make([]byte, 0, len(ca)) + for _, c := range ca { + if c == 0x00 { + break + } + s = append(s, byte(c)) + } + return string(s) +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_darwin.go b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_darwin.go index dc88b0b06..f987bd8e7 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_darwin.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_darwin.go @@ -5,18 +5,6 @@ import ( "syscall" ) -// cstr converts a nil-terminated C string into a Go string -func cstr(ca []int8) string { - s := make([]byte, 0, len(ca)) - for _, c := range ca { - if c == 0x00 { - break - } - s = append(s, byte(c)) - } - return string(s) -} - var re = regexp.MustCompile(`\\(.)`) // unescape removes backslash-escaping. The escaped characters are not diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_freebsd.go b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_freebsd.go new file mode 100644 index 000000000..f70e9975e --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/mountinfo_freebsd.go @@ -0,0 +1,7 @@ +package fstestutil + +import "errors" + +func getMountInfo(mnt string) (*MountInfo, error) { + return nil, errors.New("FreeBSD has no useful mount information") +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/record/record.go b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/record/record.go index ed8bcd342..aa2204657 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/record/record.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/record/record.go @@ -20,8 +20,7 @@ func (w *Writes) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.W n, err := w.buf.Write(req.Data) resp.Size = n if err != nil { - // TODO hiding error - return fuse.EIO + return err } return nil } @@ -293,7 +292,7 @@ var _ = fs.NodeGetxattrer(&Getxattrs{}) func (r *Getxattrs) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { tmp := *req r.rec.RecordRequest(&tmp) - return fuse.ENODATA + return fuse.ErrNoXattr } // RecordedGetxattr returns information about the Getxattr request. @@ -318,7 +317,7 @@ var _ = fs.NodeListxattrer(&Listxattrs{}) func (r *Listxattrs) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { tmp := *req r.rec.RecordRequest(&tmp) - return fuse.ENODATA + return fuse.ErrNoXattr } // RecordedListxattr returns information about the Listxattr request. diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/testfs.go b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/testfs.go index 8b1cda730..da6265f24 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/testfs.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil/testfs.go @@ -5,6 +5,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" ) // SimpleFS is a trivial FS that just implements the Root method. @@ -27,3 +28,21 @@ func (f File) Attr() fuse.Attr { return fuse.Attr{Mode: 0666} } type Dir struct{} func (f Dir) Attr() fuse.Attr { return fuse.Attr{Mode: os.ModeDir | 0777} } + +// ChildMap is a directory with child nodes looked up from a map. +type ChildMap map[string]fs.Node + +var _ = fs.Node(ChildMap{}) +var _ = fs.NodeStringLookuper(ChildMap{}) + +func (f ChildMap) Attr() fuse.Attr { + return fuse.Attr{Mode: os.ModeDir | 0777} +} + +func (f ChildMap) Lookup(ctx context.Context, name string) (fs.Node, error) { + child, ok := f[name] + if !ok { + return nil, fuse.ENOENT + } + return child, nil +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/helpers_test.go b/Godeps/_workspace/src/bazil.org/fuse/fs/helpers_test.go new file mode 100644 index 000000000..e37877c17 --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/helpers_test.go @@ -0,0 +1,67 @@ +package fs_test + +import ( + "errors" + "flag" + "os" + "os/exec" + "path/filepath" + "testing" +) + +var childHelpers = map[string]func(){} + +type childProcess struct { + name string + fn func() +} + +var _ flag.Value = (*childProcess)(nil) + +func (c *childProcess) String() string { + return c.name +} + +func (c *childProcess) Set(s string) error { + fn, ok := childHelpers[s] + if !ok { + return errors.New("helper not found") + } + c.name = s + c.fn = fn + return nil +} + +var childMode childProcess + +func init() { + flag.Var(&childMode, "fuse.internal.child", "internal use only") +} + +// childCmd prepares a test function to be run in a subprocess, with +// childMode set to true. Caller must still call Run or Start. +// +// Re-using the test executable as the subprocess is useful because +// now test executables can e.g. be cross-compiled, transferred +// between hosts, and run in settings where the whole Go development +// environment is not installed. +func childCmd(childName string) (*exec.Cmd, error) { + // caller may set cwd, so we can't rely on relative paths + executable, err := filepath.Abs(os.Args[0]) + if err != nil { + return nil, err + } + cmd := exec.Command(executable, "-fuse.internal.child="+childName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd, nil +} + +func TestMain(m *testing.M) { + flag.Parse() + if childMode.fn != nil { + childMode.fn() + os.Exit(0) + } + os.Exit(m.Run()) +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/serve.go b/Godeps/_workspace/src/bazil.org/fuse/fs/serve.go index 8b89e00e7..5247c18f4 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/serve.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/serve.go @@ -101,6 +101,15 @@ type NodeGetattrer interface { type NodeSetattrer interface { // Setattr sets the standard metadata for the receiver. + // + // Note, this is also used to communicate changes in the size of + // the file. Not implementing Setattr causes writes to be unable + // to grow the file (except with OpenDirectIO, which bypasses that + // mechanism). + // + // req.Valid is a bitmask of what fields are actually being set. + // For example, the method should not change the mode of the file + // unless req.Valid.Mode() is true. Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error } @@ -203,9 +212,7 @@ type NodeGetxattrer interface { // Getxattr gets an extended attribute by the given name from the // node. // - // If there is no xattr by that name, returns fuse.ENODATA. This - // will be translated to the platform-specific correct error code - // by the framework. + // If there is no xattr by that name, returns fuse.ErrNoXattr. Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error } @@ -223,9 +230,7 @@ type NodeSetxattrer interface { type NodeRemovexattrer interface { // Removexattr removes an extended attribute for the name. // - // If there is no xattr by that name, returns fuse.ENODATA. This - // will be translated to the platform-specific correct error code - // by the framework. + // If there is no xattr by that name, returns fuse.ErrNoXattr. Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error } @@ -279,10 +284,29 @@ type HandleReadDirAller interface { } type HandleReader interface { + // Read requests to read data from the handle. + // + // There is a page cache in the kernel that normally submits only + // page-aligned reads spanning one or more pages. However, you + // should not rely on this. To see individual requests as + // submitted by the file system clients, set OpenDirectIO. + // + // Note that reads beyond the size of the file as reported by Attr + // are not even attempted (except in OpenDirectIO mode). Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error } type HandleWriter interface { + // Write requests to write data into the handle. + // + // There is a writeback page cache in the kernel that normally submits + // only page-aligned writes spanning one or more pages. However, + // you should not rely on this. To see individual requests as + // submitted by the file system clients, set OpenDirectIO. + // + // Note that file size changes are communicated through Setattr. + // Writes beyond the size of the file as reported by Attr are not + // even attempted (except in OpenDirectIO mode). Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error } diff --git a/Godeps/_workspace/src/bazil.org/fuse/fs/serve_test.go b/Godeps/_workspace/src/bazil.org/fuse/fs/serve_test.go index 0026cad09..c39f3f57d 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fs/serve_test.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fs/serve_test.go @@ -3,14 +3,11 @@ package fs_test import ( "bytes" "errors" - "flag" "io" "io/ioutil" "log" "os" "os/exec" - "path/filepath" - "regexp" "runtime" "strings" "syscall" @@ -40,55 +37,6 @@ func init() { fstestutil.DebugByDefault() } -var childMode bool - -func init() { - flag.BoolVar(&childMode, "fuse.internal.childmode", false, "internal use only") -} - -// childCmd prepares a test function to be run in a subprocess, with -// childMode set to true. Caller must still call Run or Start. -// -// Re-using the test executable as the subprocess is useful because -// now test executables can e.g. be cross-compiled, transferred -// between hosts, and run in settings where the whole Go development -// environment is not installed. -func childCmd(testName string) (*exec.Cmd, error) { - // caller may set cwd, so we can't rely on relative paths - executable, err := filepath.Abs(os.Args[0]) - if err != nil { - return nil, err - } - testName = regexp.QuoteMeta(testName) - cmd := exec.Command(executable, "-test.run=^"+testName+"$", "-fuse.internal.childmode") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd, nil -} - -// childMapFS is an FS with one fixed child named "child". -type childMapFS map[string]fs.Node - -var _ = fs.FS(childMapFS{}) -var _ = fs.Node(childMapFS{}) -var _ = fs.NodeStringLookuper(childMapFS{}) - -func (f childMapFS) Attr() fuse.Attr { - return fuse.Attr{Inode: 1, Mode: os.ModeDir | 0777} -} - -func (f childMapFS) Root() (fs.Node, error) { - return f, nil -} - -func (f childMapFS) Lookup(ctx context.Context, name string) (fs.Node, error) { - child, ok := f[name] - if !ok { - return nil, fuse.ENOENT - } - return child, nil -} - // symlink can be embedded in a struct to make it look like a symlink. type symlink struct { target string @@ -271,7 +219,7 @@ func testReadAll(t *testing.T, path string) { func TestReadAll(t *testing.T) { t.Parallel() - mnt, err := fstestutil.MountedT(t, childMapFS{"child": readAll{}}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": readAll{}}}) if err != nil { t.Fatal(err) } @@ -300,7 +248,7 @@ func (readWithHandleRead) Read(ctx context.Context, req *fuse.ReadRequest, resp func TestReadAllWithHandleRead(t *testing.T) { t.Parallel() - mnt, err := fstestutil.MountedT(t, childMapFS{"child": readWithHandleRead{}}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": readWithHandleRead{}}}) if err != nil { t.Fatal(err) } @@ -319,7 +267,7 @@ type release struct { func TestRelease(t *testing.T) { t.Parallel() r := &release{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": r}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": r}}) if err != nil { t.Fatal(err) } @@ -346,7 +294,7 @@ type write struct { func TestWrite(t *testing.T) { t.Parallel() w := &write{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": w}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}}) if err != nil { t.Fatal(err) } @@ -393,7 +341,7 @@ type writeLarge struct { func TestWriteLarge(t *testing.T) { t.Parallel() w := &write{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": w}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}}) if err != nil { t.Fatal(err) } @@ -440,7 +388,7 @@ type writeTruncateFlush struct { func TestWriteTruncateFlush(t *testing.T) { t.Parallel() w := &writeTruncateFlush{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": w}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}}) if err != nil { t.Fatal(err) } @@ -838,7 +786,7 @@ func (dataHandleTest) Open(ctx context.Context, req *fuse.OpenRequest, resp *fus func TestDataHandle(t *testing.T) { t.Parallel() f := &dataHandleTest{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -883,7 +831,7 @@ func TestInterrupt(t *testing.T) { t.Parallel() f := &interrupt{} f.hanging = make(chan struct{}, 1) - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -947,7 +895,7 @@ type truncate struct { func testTruncate(t *testing.T, toSize int64) { t.Parallel() f := &truncate{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -988,7 +936,7 @@ type ftruncate struct { func testFtruncate(t *testing.T, toSize int64) { t.Parallel() f := &ftruncate{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1038,7 +986,7 @@ type truncateWithOpen struct { func TestTruncateWithOpen(t *testing.T) { t.Parallel() f := &truncateWithOpen{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1165,7 +1113,7 @@ func (f *chmod) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus func TestChmod(t *testing.T) { t.Parallel() f := &chmod{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1199,7 +1147,7 @@ func (f *open) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR func TestOpen(t *testing.T) { t.Parallel() f := &open{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1307,7 +1255,7 @@ func (f *getxattr) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp func TestGetxattr(t *testing.T) { t.Parallel() f := &getxattr{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1343,7 +1291,7 @@ func (f *getxattrTooSmall) Getxattr(ctx context.Context, req *fuse.GetxattrReque func TestGetxattrTooSmall(t *testing.T) { t.Parallel() f := &getxattrTooSmall{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1374,7 +1322,7 @@ func (f *getxattrSize) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, func TestGetxattrSize(t *testing.T) { t.Parallel() f := &getxattrSize{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1406,7 +1354,7 @@ func (f *listxattr) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, r func TestListxattr(t *testing.T) { t.Parallel() f := &listxattr{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1445,7 +1393,7 @@ func (f *listxattrTooSmall) Listxattr(ctx context.Context, req *fuse.ListxattrRe func TestListxattrTooSmall(t *testing.T) { t.Parallel() f := &listxattrTooSmall{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1476,7 +1424,7 @@ func (f *listxattrSize) Listxattr(ctx context.Context, req *fuse.ListxattrReques func TestListxattrSize(t *testing.T) { t.Parallel() f := &listxattrSize{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1507,7 +1455,7 @@ func testSetxattr(t *testing.T, size int) { t.Parallel() f := &setxattr{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1560,7 +1508,7 @@ type removexattr struct { func TestRemovexattr(t *testing.T) { t.Parallel() f := &removexattr{} - mnt, err := fstestutil.MountedT(t, childMapFS{"child": f}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}}) if err != nil { t.Fatal(err) } @@ -1681,6 +1629,54 @@ func (f *inMemoryFile) Write(ctx context.Context, req *fuse.WriteRequest, resp * return nil } +const mmapSize = 16 * 4096 + +var mmapWrites = map[int]byte{ + 10: 'a', + 4096: 'b', + 4097: 'c', + mmapSize - 4096: 'd', + mmapSize - 1: 'z', +} + +func helperMmap() { + f, err := os.Create("child") + if err != nil { + log.Fatalf("Create: %v", err) + } + defer f.Close() + + data, err := syscall.Mmap(int(f.Fd()), 0, mmapSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) + if err != nil { + log.Fatalf("Mmap: %v", err) + } + + for i, b := range mmapWrites { + data[i] = b + } + + if err := syscallx.Msync(data, syscall.MS_SYNC); err != nil { + log.Fatalf("Msync: %v", err) + } + + if err := syscall.Munmap(data); err != nil { + log.Fatalf("Munmap: %v", err) + } + + if err := f.Sync(); err != nil { + log.Fatalf("Fsync = %v", err) + } + + err = f.Close() + if err != nil { + log.Fatalf("Close: %v", err) + } +} + +func init() { + childHelpers["mmap"] = helperMmap +} + type mmap struct { inMemoryFile // We don't actually care about whether the fsync happened or not; @@ -1690,65 +1686,21 @@ type mmap struct { } func TestMmap(t *testing.T) { - const size = 16 * 4096 - writes := map[int]byte{ - 10: 'a', - 4096: 'b', - 4097: 'c', - size - 4096: 'd', - size - 1: 'z', + + w := &mmap{} + w.data = make([]byte, mmapSize) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}}) + if err != nil { + t.Fatal(err) } + defer mnt.Close() // Run the mmap-using parts of the test in a subprocess, to avoid // an intentional page fault hanging the whole process (because it // would need to be served by the same process, and there might // not be a thread free to do that). Merely bumping GOMAXPROCS is // not enough to prevent the hangs reliably. - if childMode { - f, err := os.Create("child") - if err != nil { - t.Fatalf("Create: %v", err) - } - defer f.Close() - - data, err := syscall.Mmap(int(f.Fd()), 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) - if err != nil { - t.Fatalf("Mmap: %v", err) - } - - for i, b := range writes { - data[i] = b - } - - if err := syscallx.Msync(data, syscall.MS_SYNC); err != nil { - t.Fatalf("Msync: %v", err) - } - - if err := syscall.Munmap(data); err != nil { - t.Fatalf("Munmap: %v", err) - } - - if err := f.Sync(); err != nil { - t.Fatalf("Fsync = %v", err) - } - - err = f.Close() - if err != nil { - t.Fatalf("Close: %v", err) - } - - return - } - - w := &mmap{} - w.data = make([]byte, size) - mnt, err := fstestutil.MountedT(t, childMapFS{"child": w}) - if err != nil { - t.Fatal(err) - } - defer mnt.Close() - - child, err := childCmd("TestMmap") + child, err := childCmd("mmap") if err != nil { t.Fatal(err) } @@ -1758,12 +1710,12 @@ func TestMmap(t *testing.T) { } got := w.data - if g, e := len(got), size; g != e { + if g, e := len(got), mmapSize; g != e { t.Fatalf("bad write length: %d != %d", g, e) } for i, g := range got { // default '\x00' for writes[i] is good here - if e := writes[i]; g != e { + if e := mmapWrites[i]; g != e { t.Errorf("wrong byte at offset %d: %q != %q", i, g, e) } } @@ -1790,7 +1742,7 @@ func (directRead) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.Re func TestDirectRead(t *testing.T) { t.Parallel() - mnt, err := fstestutil.MountedT(t, childMapFS{"child": directRead{}}) + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": directRead{}}}) if err != nil { t.Fatal(err) } @@ -1798,3 +1750,50 @@ func TestDirectRead(t *testing.T) { testReadAll(t, mnt.Dir+"/child") } + +// Test direct Write. + +type directWrite struct { + fstestutil.File + record.Writes +} + +// explicitly not defining Attr / Setattr and managing Size + +func (f *directWrite) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + // do not allow the kernel to use page cache + resp.Flags |= fuse.OpenDirectIO + return f, nil +} + +func TestDirectWrite(t *testing.T) { + t.Parallel() + w := &directWrite{} + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}}) + if err != nil { + t.Fatal(err) + } + defer mnt.Close() + + f, err := os.OpenFile(mnt.Dir+"/child", os.O_RDWR, 0666) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer f.Close() + n, err := f.Write([]byte(hi)) + if err != nil { + t.Fatalf("Write: %v", err) + } + if n != len(hi) { + t.Fatalf("short write; n=%d; hi=%d", n, len(hi)) + } + + err = f.Close() + if err != nil { + t.Fatalf("Close: %v", err) + } + + if got := string(w.RecordedWriteData()); got != hi { + t.Errorf("write = %q, want %q", got, hi) + } +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/fuse.go b/Godeps/_workspace/src/bazil.org/fuse/fuse.go index 7b5244479..a642309b9 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/fuse.go +++ b/Godeps/_workspace/src/bazil.org/fuse/fuse.go @@ -47,11 +47,17 @@ // // Op(ctx context.Context, req *OpRequest, resp *OpResponse) Error // -// where Op is the name of a FUSE operation. Op reads request parameters -// from req and writes results to resp. An operation whose only result is -// the error result omits the resp parameter. Multiple goroutines may call -// service methods simultaneously; the methods being called are responsible -// for appropriate synchronization. +// where Op is the name of a FUSE operation. Op reads request +// parameters from req and writes results to resp. An operation whose +// only result is the error result omits the resp parameter. +// +// Multiple goroutines may call service methods simultaneously; the +// methods being called are responsible for appropriate +// synchronization. +// +// The operation must not hold on to the request or response, +// including any []byte fields such as WriteRequest.Data or +// SetxattrRequest.Xattr. // // Errors // @@ -79,9 +85,12 @@ // // Authentication // -// All requests types embed a Header, meaning that the method can inspect -// req.Pid, req.Uid, and req.Gid as necessary to implement permission checking. -// Alternately, XXX. +// All requests types embed a Header, meaning that the method can +// inspect req.Pid, req.Uid, and req.Gid as necessary to implement +// permission checking. The kernel FUSE layer normally prevents other +// users from accessing the FUSE file system (to change this, see +// AllowOther, AllowRoot), but does not enforce access modes (to +// change this, see DefaultPermissions). // // Mount Options // @@ -90,8 +99,6 @@ // package fuse -// BUG(rsc): The mount code for FreeBSD has not been written yet. - import ( "bytes" "encoding/json" @@ -241,7 +248,6 @@ const ( // See also fs.Intr. EINTR = Errno(syscall.EINTR) - ENODATA = Errno(syscall.ENODATA) ERANGE = Errno(syscall.ERANGE) ENOTSUP = Errno(syscall.ENOTSUP) EEXIST = Errno(syscall.EEXIST) @@ -252,14 +258,13 @@ const ( const DefaultErrno = EIO var errnoNames = map[Errno]string{ - ENOSYS: "ENOSYS", - ESTALE: "ESTALE", - ENOENT: "ENOENT", - EIO: "EIO", - EPERM: "EPERM", - EINTR: "EINTR", - ENODATA: "ENODATA", - EEXIST: "EEXIST", + ENOSYS: "ENOSYS", + ESTALE: "ESTALE", + ENOENT: "ENOENT", + EIO: "EIO", + EPERM: "EPERM", + EINTR: "EINTR", + EEXIST: "EEXIST", } // Errno implements Error and ErrorNumber using a syscall.Errno. @@ -1115,9 +1120,6 @@ func (a *Attr) attr() (out attr) { out.Mode |= syscall.S_ISGID } out.Nlink = a.Nlink - if out.Nlink < 1 { - out.Nlink = 1 - } out.Uid = a.Uid out.Gid = a.Gid out.Rdev = a.Rdev @@ -1195,11 +1197,6 @@ func (r *GetxattrRequest) Respond(resp *GetxattrResponse) { } } -func (r *GetxattrRequest) RespondError(err error) { - err = translateGetxattrError(err) - r.Header.RespondError(err) -} - // A GetxattrResponse is the response to a GetxattrRequest. type GetxattrResponse struct { Xattr []byte @@ -1271,11 +1268,6 @@ func (r *RemovexattrRequest) Respond() { r.respond(out, unsafe.Sizeof(*out)) } -func (r *RemovexattrRequest) RespondError(err error) { - err = translateGetxattrError(err) - r.Header.RespondError(err) -} - // A SetxattrRequest asks to set an extended attribute associated with a file. type SetxattrRequest struct { Header `json:"-"` @@ -1321,11 +1313,6 @@ func (r *SetxattrRequest) Respond() { r.respond(out, unsafe.Sizeof(*out)) } -func (r *SetxattrRequest) RespondError(err error) { - err = translateGetxattrError(err) - r.Header.RespondError(err) -} - // A LookupRequest asks to look up the given name in the directory named by r.Node. type LookupRequest struct { Header `json:"-"` diff --git a/Godeps/_workspace/src/bazil.org/fuse/fuse_kernel_freebsd.go b/Godeps/_workspace/src/bazil.org/fuse/fuse_kernel_freebsd.go new file mode 100644 index 000000000..7636878c3 --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/fuse_kernel_freebsd.go @@ -0,0 +1,60 @@ +package fuse + +import "time" + +type attr struct { + Ino uint64 + Size uint64 + Blocks uint64 + Atime uint64 + Mtime uint64 + Ctime uint64 + AtimeNsec uint32 + MtimeNsec uint32 + CtimeNsec uint32 + Mode uint32 + Nlink uint32 + Uid uint32 + Gid uint32 + Rdev uint32 +} + +func (a *attr) Crtime() time.Time { + return time.Time{} +} + +func (a *attr) SetCrtime(s uint64, ns uint32) { + // ignored on freebsd +} + +func (a *attr) SetFlags(f uint32) { + // ignored on freebsd +} + +type setattrIn struct { + setattrInCommon +} + +func (in *setattrIn) BkupTime() time.Time { + return time.Time{} +} + +func (in *setattrIn) Chgtime() time.Time { + return time.Time{} +} + +func (in *setattrIn) Flags() uint32 { + return 0 +} + +func openFlags(flags uint32) OpenFlags { + return OpenFlags(flags) +} + +type getxattrIn struct { + getxattrInCommon +} + +type setxattrIn struct { + setxattrInCommon +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/mount_darwin.go b/Godeps/_workspace/src/bazil.org/fuse/mount_darwin.go index 6253ce82d..742b084c2 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/mount_darwin.go +++ b/Godeps/_workspace/src/bazil.org/fuse/mount_darwin.go @@ -59,7 +59,7 @@ func callMount(dir string, conf *MountConfig, f *os.File, ready chan<- struct{}, if strings.Contains(k, ",") || strings.Contains(v, ",") { // Silly limitation but the mount helper does not // understand any escaping. See TestMountOptionCommaError. - return fmt.Errorf("mount options cannot contain commas on OS X: %q=%q", k, v) + return fmt.Errorf("mount options cannot contain commas on darwin: %q=%q", k, v) } } cmd := exec.Command( @@ -89,7 +89,7 @@ func callMount(dir string, conf *MountConfig, f *os.File, ready chan<- struct{}, return err } go func() { - err = cmd.Wait() + err := cmd.Wait() if err != nil { if buf.Len() > 0 { output := buf.Bytes() diff --git a/Godeps/_workspace/src/bazil.org/fuse/mount_freebsd.go b/Godeps/_workspace/src/bazil.org/fuse/mount_freebsd.go new file mode 100644 index 000000000..951dcf10c --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/mount_freebsd.go @@ -0,0 +1,41 @@ +package fuse + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +func mount(dir string, conf *MountConfig, ready chan<- struct{}, errp *error) (*os.File, error) { + for k, v := range conf.options { + if strings.Contains(k, ",") || strings.Contains(v, ",") { + // Silly limitation but the mount helper does not + // understand any escaping. See TestMountOptionCommaError. + return nil, fmt.Errorf("mount options cannot contain commas on FreeBSD: %q=%q", k, v) + } + } + + f, err := os.OpenFile("/dev/fuse", os.O_RDWR, 0000) + if err != nil { + *errp = err + return nil, err + } + + cmd := exec.Command( + "/sbin/mount_fusefs", + "--safe", + "-o", conf.getOptions(), + "3", + dir, + ) + cmd.ExtraFiles = []*os.File{f} + + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("mount_fusefs: %q, %v", out, err) + } + + close(ready) + return f, nil +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options.go b/Godeps/_workspace/src/bazil.org/fuse/options.go index 643a9492d..5a0b7f180 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/options.go +++ b/Godeps/_workspace/src/bazil.org/fuse/options.go @@ -5,6 +5,10 @@ import ( "strings" ) +func dummyOption(conf *MountConfig) error { + return nil +} + // MountConfig holds the configuration for a mount operation. // Use it by passing MountOption values to Mount. type MountConfig struct { @@ -37,6 +41,8 @@ type MountOption func(*MountConfig) error // FSName sets the file system name (also called source) that is // visible in the list of mounted file systems. +// +// FreeBSD ignores this option. func FSName(name string) MountOption { return func(conf *MountConfig) error { conf.options["fsname"] = name @@ -49,6 +55,7 @@ func FSName(name string) MountOption { // `fuse.foo`. // // OS X ignores this option. +// FreeBSD ignores this option. func Subtype(fstype string) MountOption { return func(conf *MountConfig) error { conf.options["subtype"] = fstype @@ -89,6 +96,8 @@ func AllowOther() MountOption { // AllowRoot allows other users to access the file system. // // Only one of AllowOther or AllowRoot can be used. +// +// FreeBSD ignores this option. func AllowRoot() MountOption { return func(conf *MountConfig) error { if _, ok := conf.options["allow_other"]; ok { @@ -98,3 +107,26 @@ func AllowRoot() MountOption { return nil } } + +// DefaultPermissions makes the kernel enforce access control based on +// the file mode (as in chmod). +// +// Without this option, the Node itself decides what is and is not +// allowed. This is normally ok because FUSE file systems cannot be +// accessed by other users without AllowOther/AllowRoot. +// +// FreeBSD ignores this option. +func DefaultPermissions() MountOption { + return func(conf *MountConfig) error { + conf.options["default_permissions"] = "" + return nil + } +} + +// ReadOnly makes the mount read-only. +func ReadOnly() MountOption { + return func(conf *MountConfig) error { + conf.options["ro"] = "" + return nil + } +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_darwin_test.go b/Godeps/_workspace/src/bazil.org/fuse/options_darwin_test.go deleted file mode 100644 index be031f606..000000000 --- a/Godeps/_workspace/src/bazil.org/fuse/options_darwin_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package fuse_test - -import ( - "testing" - - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" -) - -func TestMountOptionCommaError(t *testing.T) { - t.Parallel() - // this test is not tied to FSName, but needs just some option - // with string content - var name = "FuseTest,Marker" - mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}}, - fuse.FSName(name), - ) - switch { - case err == nil: - mnt.Close() - t.Fatal("expected an error about commas") - case err.Error() == `mount options cannot contain commas on OS X: "fsname"="FuseTest,Marker"`: - // all good - default: - t.Fatalf("expected an error about commas, got: %v", err) - } -} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_freebsd.go b/Godeps/_workspace/src/bazil.org/fuse/options_freebsd.go new file mode 100644 index 000000000..8eb07e009 --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/options_freebsd.go @@ -0,0 +1,9 @@ +package fuse + +func localVolume(conf *MountConfig) error { + return nil +} + +func volumeName(name string) MountOption { + return dummyOption +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_helper_test.go b/Godeps/_workspace/src/bazil.org/fuse/options_helper_test.go new file mode 100644 index 000000000..f9c90e8bf --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/options_helper_test.go @@ -0,0 +1,6 @@ +package fuse + +// for TestMountOptionCommaError +func ForTestSetMountOption(conf *MountConfig, k, v string) { + conf.options[k] = v +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_linux.go b/Godeps/_workspace/src/bazil.org/fuse/options_linux.go index 69dd406ba..8eb07e009 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/options_linux.go +++ b/Godeps/_workspace/src/bazil.org/fuse/options_linux.go @@ -1,9 +1,5 @@ package fuse -func dummyOption(conf *MountConfig) error { - return nil -} - func localVolume(conf *MountConfig) error { return nil } diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_nocomma_test.go b/Godeps/_workspace/src/bazil.org/fuse/options_nocomma_test.go new file mode 100644 index 000000000..67fb66191 --- /dev/null +++ b/Godeps/_workspace/src/bazil.org/fuse/options_nocomma_test.go @@ -0,0 +1,34 @@ +// This file contains tests for platforms that have no escape +// mechanism for including commas in mount options. +// +// +build darwin + +package fuse_test + +import ( + "runtime" + "testing" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" +) + +func TestMountOptionCommaError(t *testing.T) { + t.Parallel() + // this test is not tied to any specific option, it just needs + // some string content + var evil = "FuseTest,Marker" + mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}}, + func(conf *fuse.MountConfig) error { + fuse.ForTestSetMountOption(conf, "fusetest", evil) + return nil + }, + ) + if err == nil { + mnt.Close() + t.Fatal("expected an error about commas") + } + if g, e := err.Error(), `mount options cannot contain commas on `+runtime.GOOS+`: "fusetest"="FuseTest,Marker"`; g != e { + t.Fatalf("wrong error: %q != %q", g, e) + } +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/options_test.go b/Godeps/_workspace/src/bazil.org/fuse/options_test.go index 91d615d93..59fae1379 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/options_test.go +++ b/Godeps/_workspace/src/bazil.org/fuse/options_test.go @@ -1,11 +1,15 @@ package fuse_test import ( + "os" "runtime" + "syscall" "testing" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" ) func init() { @@ -13,6 +17,9 @@ func init() { } func TestMountOptionFSName(t *testing.T) { + if runtime.GOOS == "freebsd" { + t.Skip("FreeBSD does not support FSName") + } t.Parallel() const name = "FuseTestMarker" mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}}, @@ -33,6 +40,9 @@ func TestMountOptionFSName(t *testing.T) { } func testMountOptionFSNameEvil(t *testing.T, evil string) { + if runtime.GOOS == "freebsd" { + t.Skip("FreeBSD does not support FSName") + } t.Parallel() var name = "FuseTest" + evil + "Marker" mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}}, @@ -87,6 +97,9 @@ func TestMountOptionSubtype(t *testing.T) { if runtime.GOOS == "darwin" { t.Skip("OS X does not support Subtype") } + if runtime.GOOS == "freebsd" { + t.Skip("FreeBSD does not support Subtype") + } t.Parallel() const name = "FuseTestMarker" mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}}, @@ -139,3 +152,75 @@ func TestMountOptionAllowRootThenAllowOther(t *testing.T) { t.Fatalf("wrong error: %v != %v", g, e) } } + +type unwritableFile struct{} + +func (f unwritableFile) Attr() fuse.Attr { return fuse.Attr{Mode: 0000} } + +func TestMountOptionDefaultPermissions(t *testing.T) { + if runtime.GOOS == "freebsd" { + t.Skip("FreeBSD does not support DefaultPermissions") + } + t.Parallel() + mnt, err := fstestutil.MountedT(t, + fstestutil.SimpleFS{ + fstestutil.ChildMap{"child": unwritableFile{}}, + }, + fuse.DefaultPermissions(), + ) + + if err != nil { + t.Fatal(err) + } + defer mnt.Close() + + // This will be prevented by kernel-level access checking when + // DefaultPermissions is used. + f, err := os.OpenFile(mnt.Dir+"/child", os.O_WRONLY, 0000) + if err == nil { + f.Close() + t.Fatal("expected an error") + } + if !os.IsPermission(err) { + t.Fatalf("expected a permission error, got %T: %v", err, err) + } +} + +type createrDir struct { + fstestutil.Dir +} + +var _ fs.NodeCreater = createrDir{} + +func (createrDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + // pick a really distinct error, to identify it later + return nil, nil, fuse.Errno(syscall.ENAMETOOLONG) +} + +func TestMountOptionReadOnly(t *testing.T) { + t.Parallel() + mnt, err := fstestutil.MountedT(t, + fstestutil.SimpleFS{createrDir{}}, + fuse.ReadOnly(), + ) + + if err != nil { + t.Fatal(err) + } + defer mnt.Close() + + // This will be prevented by kernel-level access checking when + // ReadOnly is used. + f, err := os.Create(mnt.Dir + "/child") + if err == nil { + f.Close() + t.Fatal("expected an error") + } + perr, ok := err.(*os.PathError) + if !ok { + t.Fatalf("expected PathError, got %T: %v", err, err) + } + if perr.Err != syscall.EROFS { + t.Fatalf("expected EROFS, got %T: %v", err, err) + } +} diff --git a/Godeps/_workspace/src/bazil.org/fuse/syscallx/syscallx_std.go b/Godeps/_workspace/src/bazil.org/fuse/syscallx/syscallx_std.go index 57353a53e..c0187a6b8 100644 --- a/Godeps/_workspace/src/bazil.org/fuse/syscallx/syscallx_std.go +++ b/Godeps/_workspace/src/bazil.org/fuse/syscallx/syscallx_std.go @@ -3,24 +3,24 @@ package syscallx // This file just contains wrappers for platforms that already have -// the right stuff in stdlib. +// the right stuff in golang.org/x/sys/unix. import ( - "syscall" + "golang.org/x/sys/unix" ) func Getxattr(path string, attr string, dest []byte) (sz int, err error) { - return syscall.Getxattr(path, attr, dest) + return unix.Getxattr(path, attr, dest) } func Listxattr(path string, dest []byte) (sz int, err error) { - return syscall.Listxattr(path, dest) + return unix.Listxattr(path, dest) } func Setxattr(path string, attr string, data []byte, flags int) (err error) { - return syscall.Setxattr(path, attr, data, flags) + return unix.Setxattr(path, attr, data, flags) } func Removexattr(path string, attr string) (err error) { - return syscall.Removexattr(path, attr) + return unix.Removexattr(path, attr) } diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md b/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md index e755e7c18..512f26e5e 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/README.md @@ -82,7 +82,7 @@ func init() { // Use the Airbrake hook to report errors that have Error severity or above to // an exception tracker. You can create custom hooks, see the Hooks section. - log.AddHook(&logrus_airbrake.AirbrakeHook{}) + log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development")) // Output to stderr instead of stdout, could also be a file. log.SetOutput(os.Stderr) @@ -164,43 +164,8 @@ You can add hooks for logging levels. For example to send errors to an exception tracking service on `Error`, `Fatal` and `Panic`, info to StatsD or log to multiple places simultaneously, e.g. syslog. -```go -// Not the real implementation of the Airbrake hook. Just a simple sample. -import ( - log "github.com/Sirupsen/logrus" -) - -func init() { - log.AddHook(new(AirbrakeHook)) -} - -type AirbrakeHook struct{} - -// `Fire()` takes the entry that the hook is fired for. `entry.Data[]` contains -// the fields for the entry. See the Fields section of the README. -func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error { - err := airbrake.Notify(entry.Data["error"].(error)) - if err != nil { - log.WithFields(log.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Info("Failed to send error to Airbrake") - } - - return nil -} - -// `Levels()` returns a slice of `Levels` the hook is fired for. -func (hook *AirbrakeHook) Levels() []log.Level { - return []log.Level{ - log.ErrorLevel, - log.FatalLevel, - log.PanicLevel, - } -} -``` - -Logrus comes with built-in hooks. Add those, or your custom hook, in `init`: +Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in +`init`: ```go import ( @@ -211,7 +176,7 @@ import ( ) func init() { - log.AddHook(new(logrus_airbrake.AirbrakeHook)) + log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development")) hook, err := logrus_syslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, "") if err != nil { @@ -233,6 +198,9 @@ func init() { Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. +* [`github.com/Sirupsen/logrus/hooks/bugsnag`](https://github.com/Sirupsen/logrus/blob/master/hooks/bugsnag/bugsnag.go) + Send errors to the Bugsnag exception tracking service. + * [`github.com/nubo/hiprus`](https://github.com/nubo/hiprus) Send errors to a channel in hipchat. @@ -321,6 +289,11 @@ The built-in logging formatters are: field to `true`. To force no colored output even if there is a TTY set the `DisableColors` field to `true` * `logrus.JSONFormatter`. Logs fields as JSON. +* `logrus_logstash.LogstashFormatter`. Logs fields as Logstash Events (http://logstash.net). + + ```go + logrus.SetFormatter(&logrus_logstash.LogstashFormatter{Type: “application_name"}) + ``` Third party logging formatters: diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/basic/basic.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/basic/basic.go index 19422a212..ae3fe7d13 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/basic/basic.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/basic/basic.go @@ -9,6 +9,7 @@ var log = logrus.New() func init() { log.Formatter = new(logrus.JSONFormatter) log.Formatter = new(logrus.TextFormatter) // default + log.Level = logrus.DebugLevel } func main() { @@ -23,6 +24,11 @@ func main() { } }() + log.WithFields(logrus.Fields{ + "animal": "walrus", + "number": 8, + }).Debug("Started observing beach") + log.WithFields(logrus.Fields{ "animal": "walrus", "size": 10, @@ -33,6 +39,10 @@ func main() { "number": 122, }).Warn("The group's number increased tremendously!") + log.WithFields(logrus.Fields{ + "temperature": -4, + }).Debug("Temperature changes") + log.WithFields(logrus.Fields{ "animal": "orca", "size": 9009, diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/hook/hook.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/hook/hook.go index d6d43d486..b68980d62 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/hook/hook.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/examples/hook/hook.go @@ -3,21 +3,17 @@ package main import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake" - "github.com/tobi/airbrake-go" ) var log = logrus.New() func init() { - log.Formatter = new(logrus.TextFormatter) // default - log.Hooks.Add(new(logrus_airbrake.AirbrakeHook)) + log.Formatter = // default + new(logrus.TextFormatter) + log.Hooks.Add(airbrake.NewHook("https://example.com", "xyz", "development")) } func main() { - airbrake.Endpoint = "https://exceptions.whatever.com/notifier_api/v2/notices.xml" - airbrake.ApiKey = "whatever" - airbrake.Environment = "production" - log.WithFields(logrus.Fields{ "animal": "walrus", "size": 10, diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go new file mode 100644 index 000000000..6939d2e39 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go @@ -0,0 +1,48 @@ +package logstash + +import ( + "encoding/json" + "fmt" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" + "time" +) + +// Formatter generates json in logstash format. +// Logstash site: http://logstash.net/ +type LogstashFormatter struct { + Type string // if not empty use for logstash type field. +} + +func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) { + entry.Data["@version"] = 1 + entry.Data["@timestamp"] = entry.Time.Format(time.RFC3339) + + // set message field + v, ok := entry.Data["message"] + if ok { + entry.Data["fields.message"] = v + } + entry.Data["message"] = entry.Message + + // set level field + v, ok = entry.Data["level"] + if ok { + entry.Data["fields.level"] = v + } + entry.Data["level"] = entry.Level.String() + + // set type field + if f.Type != "" { + v, ok = entry.Data["type"] + if ok { + entry.Data["fields.type"] = v + } + entry.Data["type"] = f.Type + } + + serialized, err := json.Marshal(entry.Data) + if err != nil { + return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) + } + return append(serialized, '\n'), nil +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash_test.go new file mode 100644 index 000000000..0775b3792 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/formatters/logstash/logstash_test.go @@ -0,0 +1,52 @@ +package logstash + +import ( + "bytes" + "encoding/json" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestLogstashFormatter(t *testing.T) { + assert := assert.New(t) + + lf := LogstashFormatter{Type: "abc"} + + fields := logrus.Fields{ + "message": "def", + "level": "ijk", + "type": "lmn", + "one": 1, + "pi": 3.14, + "bool": true, + } + + entry := logrus.WithFields(fields) + entry.Message = "msg" + entry.Level = logrus.InfoLevel + + b, _ := lf.Format(entry) + + var data map[string]interface{} + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + dec.Decode(&data) + + // base fields + assert.Equal(json.Number("1"), data["@version"]) + assert.NotEmpty(data["@timestamp"]) + assert.Equal("abc", data["type"]) + assert.Equal("msg", data["message"]) + assert.Equal("info", data["level"]) + + // substituted fields + assert.Equal("def", data["fields.message"]) + assert.Equal("ijk", data["fields.level"]) + assert.Equal("lmn", data["fields.type"]) + + // formats + assert.Equal(json.Number("1"), data["one"]) + assert.Equal(json.Number("3.14"), data["pi"]) + assert.Equal(true, data["bool"]) +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go index 0162ddf73..16f73cac6 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go @@ -1,51 +1,51 @@ -package logrus_airbrake +package airbrake import ( + "errors" + "fmt" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" "github.com/tobi/airbrake-go" ) // AirbrakeHook to send exceptions to an exception-tracking service compatible -// with the Airbrake API. You must set: -// * airbrake.Endpoint -// * airbrake.ApiKey -// * airbrake.Environment (only sends exceptions when set to "production") -// -// Before using this hook, to send an error. Entries that trigger an Error, -// Fatal or Panic should now include an "error" field to send to Airbrake. -type AirbrakeHook struct{} +// with the Airbrake API. +type airbrakeHook struct { + APIKey string + Endpoint string + Environment string +} -func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error { - if entry.Data["error"] == nil { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Warn("Exceptions sent to Airbrake must have an 'error' key with the error") - return nil +func NewHook(endpoint, apiKey, env string) *airbrakeHook { + return &airbrakeHook{ + APIKey: apiKey, + Endpoint: endpoint, + Environment: env, } +} +func (hook *airbrakeHook) Fire(entry *logrus.Entry) error { + airbrake.ApiKey = hook.APIKey + airbrake.Endpoint = hook.Endpoint + airbrake.Environment = hook.Environment + + var notifyErr error err, ok := entry.Data["error"].(error) - if !ok { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Warn("Exceptions sent to Airbrake must have an `error` key of type `error`") - return nil + if ok { + notifyErr = err + } else { + notifyErr = errors.New(entry.Message) } - airErr := airbrake.Notify(err) + airErr := airbrake.Notify(notifyErr) if airErr != nil { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - "error": airErr, - }).Warn("Failed to send error to Airbrake") + return fmt.Errorf("Failed to send error to Airbrake: %s", airErr) } return nil } -func (hook *AirbrakeHook) Levels() []logrus.Level { +func (hook *airbrakeHook) Levels() []logrus.Level { return []logrus.Level{ logrus.ErrorLevel, logrus.FatalLevel, diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake_test.go new file mode 100644 index 000000000..266679173 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake_test.go @@ -0,0 +1,133 @@ +package airbrake + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" +) + +type notice struct { + Error NoticeError `xml:"error"` +} +type NoticeError struct { + Class string `xml:"class"` + Message string `xml:"message"` +} + +type customErr struct { + msg string +} + +func (e *customErr) Error() string { + return e.msg +} + +const ( + testAPIKey = "abcxyz" + testEnv = "development" + expectedClass = "*airbrake.customErr" + expectedMsg = "foo" + unintendedMsg = "Airbrake will not see this string" +) + +var ( + noticeError = make(chan NoticeError, 1) +) + +// TestLogEntryMessageReceived checks if invoking Logrus' log.Error +// method causes an XML payload containing the log entry message is received +// by a HTTP server emulating an Airbrake-compatible endpoint. +func TestLogEntryMessageReceived(t *testing.T) { + log := logrus.New() + ts := startAirbrakeServer(t) + defer ts.Close() + + hook := NewHook(ts.URL, testAPIKey, "production") + log.Hooks.Add(hook) + + log.Error(expectedMsg) + + select { + case received := <-noticeError: + if received.Message != expectedMsg { + t.Errorf("Unexpected message received: %s", received.Message) + } + case <-time.After(time.Second): + t.Error("Timed out; no notice received by Airbrake API") + } +} + +// TestLogEntryMessageReceived confirms that, when passing an error type using +// logrus.Fields, a HTTP server emulating an Airbrake endpoint receives the +// error message returned by the Error() method on the error interface +// rather than the logrus.Entry.Message string. +func TestLogEntryWithErrorReceived(t *testing.T) { + log := logrus.New() + ts := startAirbrakeServer(t) + defer ts.Close() + + hook := NewHook(ts.URL, testAPIKey, "production") + log.Hooks.Add(hook) + + log.WithFields(logrus.Fields{ + "error": &customErr{expectedMsg}, + }).Error(unintendedMsg) + + select { + case received := <-noticeError: + if received.Message != expectedMsg { + t.Errorf("Unexpected message received: %s", received.Message) + } + if received.Class != expectedClass { + t.Errorf("Unexpected error class: %s", received.Class) + } + case <-time.After(time.Second): + t.Error("Timed out; no notice received by Airbrake API") + } +} + +// TestLogEntryWithNonErrorTypeNotReceived confirms that, when passing a +// non-error type using logrus.Fields, a HTTP server emulating an Airbrake +// endpoint receives the logrus.Entry.Message string. +// +// Only error types are supported when setting the 'error' field using +// logrus.WithFields(). +func TestLogEntryWithNonErrorTypeNotReceived(t *testing.T) { + log := logrus.New() + ts := startAirbrakeServer(t) + defer ts.Close() + + hook := NewHook(ts.URL, testAPIKey, "production") + log.Hooks.Add(hook) + + log.WithFields(logrus.Fields{ + "error": expectedMsg, + }).Error(unintendedMsg) + + select { + case received := <-noticeError: + if received.Message != unintendedMsg { + t.Errorf("Unexpected message received: %s", received.Message) + } + case <-time.After(time.Second): + t.Error("Timed out; no notice received by Airbrake API") + } +} + +func startAirbrakeServer(t *testing.T) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var notice notice + if err := xml.NewDecoder(r.Body).Decode(¬ice); err != nil { + t.Error(err) + } + r.Body.Close() + + noticeError <- notice.Error + })) + + return ts +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go new file mode 100644 index 000000000..71df1b274 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go @@ -0,0 +1,68 @@ +package logrus_bugsnag + +import ( + "errors" + + "github.com/bugsnag/bugsnag-go" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" +) + +type bugsnagHook struct{} + +// ErrBugsnagUnconfigured is returned if NewBugsnagHook is called before +// bugsnag.Configure. Bugsnag must be configured before the hook. +var ErrBugsnagUnconfigured = errors.New("bugsnag must be configured before installing this logrus hook") + +// ErrBugsnagSendFailed indicates that the hook failed to submit an error to +// bugsnag. The error was successfully generated, but `bugsnag.Notify()` +// failed. +type ErrBugsnagSendFailed struct { + err error +} + +func (e ErrBugsnagSendFailed) Error() string { + return "failed to send error to Bugsnag: " + e.err.Error() +} + +// NewBugsnagHook initializes a logrus hook which sends exceptions to an +// exception-tracking service compatible with the Bugsnag API. Before using +// this hook, you must call bugsnag.Configure(). The returned object should be +// registered with a log via `AddHook()` +// +// Entries that trigger an Error, Fatal or Panic should now include an "error" +// field to send to Bugsnag. +func NewBugsnagHook() (*bugsnagHook, error) { + if bugsnag.Config.APIKey == "" { + return nil, ErrBugsnagUnconfigured + } + return &bugsnagHook{}, nil +} + +// Fire forwards an error to Bugsnag. Given a logrus.Entry, it extracts the +// "error" field (or the Message if the error isn't present) and sends it off. +func (hook *bugsnagHook) Fire(entry *logrus.Entry) error { + var notifyErr error + err, ok := entry.Data["error"].(error) + if ok { + notifyErr = err + } else { + notifyErr = errors.New(entry.Message) + } + + bugsnagErr := bugsnag.Notify(notifyErr) + if bugsnagErr != nil { + return ErrBugsnagSendFailed{bugsnagErr} + } + + return nil +} + +// Levels enumerates the log levels on which the error should be forwarded to +// bugsnag: everything at or above the "Error" level. +func (hook *bugsnagHook) Levels() []logrus.Level { + return []logrus.Level{ + logrus.ErrorLevel, + logrus.FatalLevel, + logrus.PanicLevel, + } +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag_test.go new file mode 100644 index 000000000..24e551ca5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag_test.go @@ -0,0 +1,64 @@ +package logrus_bugsnag + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/bugsnag/bugsnag-go" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" +) + +type notice struct { + Events []struct { + Exceptions []struct { + Message string `json:"message"` + } `json:"exceptions"` + } `json:"events"` +} + +func TestNoticeReceived(t *testing.T) { + msg := make(chan string, 1) + expectedMsg := "foo" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var notice notice + data, _ := ioutil.ReadAll(r.Body) + if err := json.Unmarshal(data, ¬ice); err != nil { + t.Error(err) + } + _ = r.Body.Close() + + msg <- notice.Events[0].Exceptions[0].Message + })) + defer ts.Close() + + hook := &bugsnagHook{} + + bugsnag.Configure(bugsnag.Configuration{ + Endpoint: ts.URL, + ReleaseStage: "production", + APIKey: "12345678901234567890123456789012", + Synchronous: true, + }) + + log := logrus.New() + log.Hooks.Add(hook) + + log.WithFields(logrus.Fields{ + "error": errors.New(expectedMsg), + }).Error("Bugsnag will not see this string") + + select { + case received := <-msg: + if received != expectedMsg { + t.Errorf("Unexpected message received: %s", received) + } + case <-time.After(time.Second): + t.Error("Timed out; no notice received by Bugsnag API") + } +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md index a409f3b04..19e58bb45 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/sentry/README.md @@ -57,5 +57,5 @@ with a call to `NewSentryHook`. This can be changed by assigning a value to the ```go hook, _ := logrus_sentry.NewSentryHook(...) -hook.Timeout = 20*time.Seconds +hook.Timeout = 20*time.Second ``` diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go index b09227c2b..5c4c44bbe 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter.go @@ -11,7 +11,14 @@ type JSONFormatter struct{} func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { data := make(Fields, len(entry.Data)+3) for k, v := range entry.Data { - data[k] = v + switch v := v.(type) { + case error: + // Otherwise errors are ignored by `encoding/json` + // https://github.com/Sirupsen/logrus/issues/137 + data[k] = v.Error() + default: + data[k] = v + } } prefixFieldClashes(data) data["time"] = entry.Time.Format(time.RFC3339) diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter_test.go new file mode 100644 index 000000000..1d7087325 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/json_formatter_test.go @@ -0,0 +1,120 @@ +package logrus + +import ( + "encoding/json" + "errors" + + "testing" +) + +func TestErrorNotLost(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("error", errors.New("wild walrus"))) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["error"] != "wild walrus" { + t.Fatal("Error field not set") + } +} + +func TestErrorNotLostOnFieldNotNamedError(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("omg", errors.New("wild walrus"))) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["omg"] != "wild walrus" { + t.Fatal("Error field not set") + } +} + +func TestFieldClashWithTime(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("time", "right now!")) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["fields.time"] != "right now!" { + t.Fatal("fields.time not set to original time field") + } + + if entry["time"] != "0001-01-01T00:00:00Z" { + t.Fatal("time field not set to current time, was: ", entry["time"]) + } +} + +func TestFieldClashWithMsg(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("msg", "something")) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["fields.msg"] != "something" { + t.Fatal("fields.msg not set to original msg field") + } +} + +func TestFieldClashWithLevel(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("level", "something")) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["fields.level"] != "something" { + t.Fatal("fields.level not set to original level field") + } +} + +func TestJSONEntryEndsWithNewline(t *testing.T) { + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("level", "something")) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + if b[len(b)-1] != '\n' { + t.Fatal("Expected JSON log entry to end with a newline") + } +} diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go index b392e547a..da928a375 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/logger.go @@ -65,11 +65,15 @@ func (logger *Logger) WithFields(fields Fields) *Entry { } func (logger *Logger) Debugf(format string, args ...interface{}) { - NewEntry(logger).Debugf(format, args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debugf(format, args...) + } } func (logger *Logger) Infof(format string, args ...interface{}) { - NewEntry(logger).Infof(format, args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Infof(format, args...) + } } func (logger *Logger) Printf(format string, args ...interface{}) { @@ -77,31 +81,45 @@ func (logger *Logger) Printf(format string, args ...interface{}) { } func (logger *Logger) Warnf(format string, args ...interface{}) { - NewEntry(logger).Warnf(format, args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnf(format, args...) + } } func (logger *Logger) Warningf(format string, args ...interface{}) { - NewEntry(logger).Warnf(format, args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnf(format, args...) + } } func (logger *Logger) Errorf(format string, args ...interface{}) { - NewEntry(logger).Errorf(format, args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Errorf(format, args...) + } } func (logger *Logger) Fatalf(format string, args ...interface{}) { - NewEntry(logger).Fatalf(format, args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatalf(format, args...) + } } func (logger *Logger) Panicf(format string, args ...interface{}) { - NewEntry(logger).Panicf(format, args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panicf(format, args...) + } } func (logger *Logger) Debug(args ...interface{}) { - NewEntry(logger).Debug(args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debug(args...) + } } func (logger *Logger) Info(args ...interface{}) { - NewEntry(logger).Info(args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Info(args...) + } } func (logger *Logger) Print(args ...interface{}) { @@ -109,31 +127,45 @@ func (logger *Logger) Print(args ...interface{}) { } func (logger *Logger) Warn(args ...interface{}) { - NewEntry(logger).Warn(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warn(args...) + } } func (logger *Logger) Warning(args ...interface{}) { - NewEntry(logger).Warn(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warn(args...) + } } func (logger *Logger) Error(args ...interface{}) { - NewEntry(logger).Error(args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Error(args...) + } } func (logger *Logger) Fatal(args ...interface{}) { - NewEntry(logger).Fatal(args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatal(args...) + } } func (logger *Logger) Panic(args ...interface{}) { - NewEntry(logger).Panic(args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panic(args...) + } } func (logger *Logger) Debugln(args ...interface{}) { - NewEntry(logger).Debugln(args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debugln(args...) + } } func (logger *Logger) Infoln(args ...interface{}) { - NewEntry(logger).Infoln(args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Infoln(args...) + } } func (logger *Logger) Println(args ...interface{}) { @@ -141,21 +173,31 @@ func (logger *Logger) Println(args ...interface{}) { } func (logger *Logger) Warnln(args ...interface{}) { - NewEntry(logger).Warnln(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnln(args...) + } } func (logger *Logger) Warningln(args ...interface{}) { - NewEntry(logger).Warnln(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnln(args...) + } } func (logger *Logger) Errorln(args ...interface{}) { - NewEntry(logger).Errorln(args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Errorln(args...) + } } func (logger *Logger) Fatalln(args ...interface{}) { - NewEntry(logger).Fatalln(args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatalln(args...) + } } func (logger *Logger) Panicln(args ...interface{}) { - NewEntry(logger).Panicln(args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panicln(args...) + } } diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/terminal_openbsd.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/terminal_openbsd.go index d238bfa0b..af609a53d 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/terminal_openbsd.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/terminal_openbsd.go @@ -1,4 +1,3 @@ - package logrus import "syscall" diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go index 8c42c74d0..0a06a1105 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter.go @@ -3,7 +3,6 @@ package logrus import ( "bytes" "fmt" - "regexp" "sort" "strings" "time" @@ -15,12 +14,12 @@ const ( green = 32 yellow = 33 blue = 34 + gray = 37 ) var ( baseTimestamp time.Time isTerminal bool - noQuoteNeeded *regexp.Regexp ) func init() { @@ -46,15 +45,22 @@ type TextFormatter struct { // Enable logging the full timestamp when a TTY is attached instead of just // the time passed since beginning of execution. FullTimestamp bool + + // The fields are sorted by default for a consistent output. For applications + // that log extremely frequently and don't use the JSON formatter this may not + // be desired. + DisableSorting bool } func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { - var keys []string = make([]string, 0, len(entry.Data)) for k := range entry.Data { keys = append(keys, k) } - sort.Strings(keys) + + if !f.DisableSorting { + sort.Strings(keys) + } b := &bytes.Buffer{} @@ -82,6 +88,8 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []string) { var levelColor int switch entry.Level { + case DebugLevel: + levelColor = gray case WarnLevel: levelColor = yellow case ErrorLevel, FatalLevel, PanicLevel: @@ -107,7 +115,7 @@ func needsQuoting(text string) bool { for _, ch := range text { if !((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || - (ch >= '0' && ch < '9') || + (ch >= '0' && ch <= '9') || ch == '-' || ch == '.') { return false } diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter_test.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter_test.go index f604f1b00..28a949907 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter_test.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/text_formatter_test.go @@ -25,9 +25,13 @@ func TestQuoting(t *testing.T) { checkQuoting(false, "abcd") checkQuoting(false, "v1.0") + checkQuoting(false, "1234567890") checkQuoting(true, "/foobar") checkQuoting(true, "x y") checkQuoting(true, "x,y") checkQuoting(false, errors.New("invalid")) checkQuoting(true, errors.New("invalid argument")) } + +// TODO add tests for sorting etc., this requires a parser for the text +// formatter output. diff --git a/Godeps/_workspace/src/github.com/Sirupsen/logrus/writer.go b/Godeps/_workspace/src/github.com/Sirupsen/logrus/writer.go index 90d3e01b4..1e30b1c75 100644 --- a/Godeps/_workspace/src/github.com/Sirupsen/logrus/writer.go +++ b/Godeps/_workspace/src/github.com/Sirupsen/logrus/writer.go @@ -6,7 +6,7 @@ import ( "runtime" ) -func (logger *Logger) Writer() (*io.PipeWriter) { +func (logger *Logger) Writer() *io.PipeWriter { reader, writer := io.Pipe() go logger.writerScanner(reader) diff --git a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go index 4e10221a1..f1f8ab797 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go +++ b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go @@ -58,6 +58,13 @@ func NewVersion(version string) (*Version, error) { return &v, nil } +func Must(v *Version, err error) *Version { + if err != nil { + panic(err) + } + return v +} + func (v *Version) String() string { var buffer bytes.Buffer diff --git a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go index de09cd7dc..9bfc3b8a9 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go +++ b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go @@ -1,7 +1,9 @@ package semver import ( + "errors" "math/rand" + "reflect" "testing" "time" ) @@ -185,3 +187,37 @@ func TestBumpPatch(t *testing.T) { t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version) } } + +func TestMust(t *testing.T) { + tests := []struct { + versionStr string + + version *Version + recov interface{} + }{ + { + versionStr: "1.0.0", + version: &Version{Major: 1}, + }, + { + versionStr: "version number", + recov: errors.New("version number is not in dotted-tri format"), + }, + } + + for _, tt := range tests { + func() { + defer func() { + recov := recover() + if !reflect.DeepEqual(tt.recov, recov) { + t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov) + } + }() + + version := Must(NewVersion(tt.versionStr)) + if !reflect.DeepEqual(tt.version, version) { + t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version) + } + }() + } +} diff --git a/Godeps/_workspace/src/github.com/gorilla/context/.travis.yml b/Godeps/_workspace/src/github.com/gorilla/context/.travis.yml deleted file mode 100644 index d87d46576..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go - -go: - - 1.0 - - 1.1 - - 1.2 - - tip diff --git a/Godeps/_workspace/src/github.com/gorilla/context/LICENSE b/Godeps/_workspace/src/github.com/gorilla/context/LICENSE deleted file mode 100644 index 0e5fb8728..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 Rodrigo Moraes. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/gorilla/context/README.md b/Godeps/_workspace/src/github.com/gorilla/context/README.md deleted file mode 100644 index c60a31b05..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/README.md +++ /dev/null @@ -1,7 +0,0 @@ -context -======= -[![Build Status](https://travis-ci.org/gorilla/context.png?branch=master)](https://travis-ci.org/gorilla/context) - -gorilla/context is a general purpose registry for global request variables. - -Read the full documentation here: http://www.gorillatoolkit.org/pkg/context diff --git a/Godeps/_workspace/src/github.com/gorilla/context/context.go b/Godeps/_workspace/src/github.com/gorilla/context/context.go deleted file mode 100644 index 81cb128b1..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/context.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package context - -import ( - "net/http" - "sync" - "time" -) - -var ( - mutex sync.RWMutex - data = make(map[*http.Request]map[interface{}]interface{}) - datat = make(map[*http.Request]int64) -) - -// Set stores a value for a given key in a given request. -func Set(r *http.Request, key, val interface{}) { - mutex.Lock() - if data[r] == nil { - data[r] = make(map[interface{}]interface{}) - datat[r] = time.Now().Unix() - } - data[r][key] = val - mutex.Unlock() -} - -// Get returns a value stored for a given key in a given request. -func Get(r *http.Request, key interface{}) interface{} { - mutex.RLock() - if ctx := data[r]; ctx != nil { - value := ctx[key] - mutex.RUnlock() - return value - } - mutex.RUnlock() - return nil -} - -// GetOk returns stored value and presence state like multi-value return of map access. -func GetOk(r *http.Request, key interface{}) (interface{}, bool) { - mutex.RLock() - if _, ok := data[r]; ok { - value, ok := data[r][key] - mutex.RUnlock() - return value, ok - } - mutex.RUnlock() - return nil, false -} - -// GetAll returns all stored values for the request as a map. Nil is returned for invalid requests. -func GetAll(r *http.Request) map[interface{}]interface{} { - mutex.RLock() - if context, ok := data[r]; ok { - result := make(map[interface{}]interface{}, len(context)) - for k, v := range context { - result[k] = v - } - mutex.RUnlock() - return result - } - mutex.RUnlock() - return nil -} - -// GetAllOk returns all stored values for the request as a map and a boolean value that indicates if -// the request was registered. -func GetAllOk(r *http.Request) (map[interface{}]interface{}, bool) { - mutex.RLock() - context, ok := data[r] - result := make(map[interface{}]interface{}, len(context)) - for k, v := range context { - result[k] = v - } - mutex.RUnlock() - return result, ok -} - -// Delete removes a value stored for a given key in a given request. -func Delete(r *http.Request, key interface{}) { - mutex.Lock() - if data[r] != nil { - delete(data[r], key) - } - mutex.Unlock() -} - -// Clear removes all values stored for a given request. -// -// This is usually called by a handler wrapper to clean up request -// variables at the end of a request lifetime. See ClearHandler(). -func Clear(r *http.Request) { - mutex.Lock() - clear(r) - mutex.Unlock() -} - -// clear is Clear without the lock. -func clear(r *http.Request) { - delete(data, r) - delete(datat, r) -} - -// Purge removes request data stored for longer than maxAge, in seconds. -// It returns the amount of requests removed. -// -// If maxAge <= 0, all request data is removed. -// -// This is only used for sanity check: in case context cleaning was not -// properly set some request data can be kept forever, consuming an increasing -// amount of memory. In case this is detected, Purge() must be called -// periodically until the problem is fixed. -func Purge(maxAge int) int { - mutex.Lock() - count := 0 - if maxAge <= 0 { - count = len(data) - data = make(map[*http.Request]map[interface{}]interface{}) - datat = make(map[*http.Request]int64) - } else { - min := time.Now().Unix() - int64(maxAge) - for r := range data { - if datat[r] < min { - clear(r) - count++ - } - } - } - mutex.Unlock() - return count -} - -// ClearHandler wraps an http.Handler and clears request values at the end -// of a request lifetime. -func ClearHandler(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer Clear(r) - h.ServeHTTP(w, r) - }) -} diff --git a/Godeps/_workspace/src/github.com/gorilla/context/context_test.go b/Godeps/_workspace/src/github.com/gorilla/context/context_test.go deleted file mode 100644 index 6ada8ec31..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/context_test.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package context - -import ( - "net/http" - "testing" -) - -type keyType int - -const ( - key1 keyType = iota - key2 -) - -func TestContext(t *testing.T) { - assertEqual := func(val interface{}, exp interface{}) { - if val != exp { - t.Errorf("Expected %v, got %v.", exp, val) - } - } - - r, _ := http.NewRequest("GET", "http://localhost:8080/", nil) - emptyR, _ := http.NewRequest("GET", "http://localhost:8080/", nil) - - // Get() - assertEqual(Get(r, key1), nil) - - // Set() - Set(r, key1, "1") - assertEqual(Get(r, key1), "1") - assertEqual(len(data[r]), 1) - - Set(r, key2, "2") - assertEqual(Get(r, key2), "2") - assertEqual(len(data[r]), 2) - - //GetOk - value, ok := GetOk(r, key1) - assertEqual(value, "1") - assertEqual(ok, true) - - value, ok = GetOk(r, "not exists") - assertEqual(value, nil) - assertEqual(ok, false) - - Set(r, "nil value", nil) - value, ok = GetOk(r, "nil value") - assertEqual(value, nil) - assertEqual(ok, true) - - // GetAll() - values := GetAll(r) - assertEqual(len(values), 3) - - // GetAll() for empty request - values = GetAll(emptyR) - if values != nil { - t.Error("GetAll didn't return nil value for invalid request") - } - - // GetAllOk() - values, ok = GetAllOk(r) - assertEqual(len(values), 3) - assertEqual(ok, true) - - // GetAllOk() for empty request - values, ok = GetAllOk(emptyR) - assertEqual(value, nil) - assertEqual(ok, false) - - // Delete() - Delete(r, key1) - assertEqual(Get(r, key1), nil) - assertEqual(len(data[r]), 2) - - Delete(r, key2) - assertEqual(Get(r, key2), nil) - assertEqual(len(data[r]), 1) - - // Clear() - Clear(r) - assertEqual(len(data), 0) -} - -func parallelReader(r *http.Request, key string, iterations int, wait, done chan struct{}) { - <-wait - for i := 0; i < iterations; i++ { - Get(r, key) - } - done <- struct{}{} - -} - -func parallelWriter(r *http.Request, key, value string, iterations int, wait, done chan struct{}) { - <-wait - for i := 0; i < iterations; i++ { - Get(r, key) - } - done <- struct{}{} - -} - -func benchmarkMutex(b *testing.B, numReaders, numWriters, iterations int) { - - b.StopTimer() - r, _ := http.NewRequest("GET", "http://localhost:8080/", nil) - done := make(chan struct{}) - b.StartTimer() - - for i := 0; i < b.N; i++ { - wait := make(chan struct{}) - - for i := 0; i < numReaders; i++ { - go parallelReader(r, "test", iterations, wait, done) - } - - for i := 0; i < numWriters; i++ { - go parallelWriter(r, "test", "123", iterations, wait, done) - } - - close(wait) - - for i := 0; i < numReaders+numWriters; i++ { - <-done - } - - } - -} - -func BenchmarkMutexSameReadWrite1(b *testing.B) { - benchmarkMutex(b, 1, 1, 32) -} -func BenchmarkMutexSameReadWrite2(b *testing.B) { - benchmarkMutex(b, 2, 2, 32) -} -func BenchmarkMutexSameReadWrite4(b *testing.B) { - benchmarkMutex(b, 4, 4, 32) -} -func BenchmarkMutex1(b *testing.B) { - benchmarkMutex(b, 2, 8, 32) -} -func BenchmarkMutex2(b *testing.B) { - benchmarkMutex(b, 16, 4, 64) -} -func BenchmarkMutex3(b *testing.B) { - benchmarkMutex(b, 1, 2, 128) -} -func BenchmarkMutex4(b *testing.B) { - benchmarkMutex(b, 128, 32, 256) -} -func BenchmarkMutex5(b *testing.B) { - benchmarkMutex(b, 1024, 2048, 64) -} -func BenchmarkMutex6(b *testing.B) { - benchmarkMutex(b, 2048, 1024, 512) -} diff --git a/Godeps/_workspace/src/github.com/gorilla/context/doc.go b/Godeps/_workspace/src/github.com/gorilla/context/doc.go deleted file mode 100644 index 73c740031..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/context/doc.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package context stores values shared during a request lifetime. - -For example, a router can set variables extracted from the URL and later -application handlers can access those values, or it can be used to store -sessions values to be saved at the end of a request. There are several -others common uses. - -The idea was posted by Brad Fitzpatrick to the go-nuts mailing list: - - http://groups.google.com/group/golang-nuts/msg/e2d679d303aa5d53 - -Here's the basic usage: first define the keys that you will need. The key -type is interface{} so a key can be of any type that supports equality. -Here we define a key using a custom int type to avoid name collisions: - - package foo - - import ( - "github.com/gorilla/context" - ) - - type key int - - const MyKey key = 0 - -Then set a variable. Variables are bound to an http.Request object, so you -need a request instance to set a value: - - context.Set(r, MyKey, "bar") - -The application can later access the variable using the same key you provided: - - func MyHandler(w http.ResponseWriter, r *http.Request) { - // val is "bar". - val := context.Get(r, foo.MyKey) - - // returns ("bar", true) - val, ok := context.GetOk(r, foo.MyKey) - // ... - } - -And that's all about the basic usage. We discuss some other ideas below. - -Any type can be stored in the context. To enforce a given type, make the key -private and wrap Get() and Set() to accept and return values of a specific -type: - - type key int - - const mykey key = 0 - - // GetMyKey returns a value for this package from the request values. - func GetMyKey(r *http.Request) SomeType { - if rv := context.Get(r, mykey); rv != nil { - return rv.(SomeType) - } - return nil - } - - // SetMyKey sets a value for this package in the request values. - func SetMyKey(r *http.Request, val SomeType) { - context.Set(r, mykey, val) - } - -Variables must be cleared at the end of a request, to remove all values -that were stored. This can be done in an http.Handler, after a request was -served. Just call Clear() passing the request: - - context.Clear(r) - -...or use ClearHandler(), which conveniently wraps an http.Handler to clear -variables at the end of a request lifetime. - -The Routers from the packages gorilla/mux and gorilla/pat call Clear() -so if you are using either of them you don't need to clear the context manually. -*/ -package context diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/.travis.yml b/Godeps/_workspace/src/github.com/gorilla/mux/.travis.yml deleted file mode 100644 index d87d46576..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go - -go: - - 1.0 - - 1.1 - - 1.2 - - tip diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/LICENSE b/Godeps/_workspace/src/github.com/gorilla/mux/LICENSE deleted file mode 100644 index 0e5fb8728..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 Rodrigo Moraes. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/README.md b/Godeps/_workspace/src/github.com/gorilla/mux/README.md deleted file mode 100644 index e60301b03..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/README.md +++ /dev/null @@ -1,7 +0,0 @@ -mux -=== -[![Build Status](https://travis-ci.org/gorilla/mux.png?branch=master)](https://travis-ci.org/gorilla/mux) - -gorilla/mux is a powerful URL router and dispatcher. - -Read the full documentation here: http://www.gorillatoolkit.org/pkg/mux diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/bench_test.go b/Godeps/_workspace/src/github.com/gorilla/mux/bench_test.go deleted file mode 100644 index c5f97b2b2..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/bench_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "net/http" - "testing" -) - -func BenchmarkMux(b *testing.B) { - router := new(Router) - handler := func(w http.ResponseWriter, r *http.Request) {} - router.HandleFunc("/v1/{v1}", handler) - - request, _ := http.NewRequest("GET", "/v1/anything", nil) - for i := 0; i < b.N; i++ { - router.ServeHTTP(nil, request) - } -} diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/doc.go b/Godeps/_workspace/src/github.com/gorilla/mux/doc.go deleted file mode 100644 index b2deed34c..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/doc.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package gorilla/mux implements a request router and dispatcher. - -The name mux stands for "HTTP request multiplexer". Like the standard -http.ServeMux, mux.Router matches incoming requests against a list of -registered routes and calls a handler for the route that matches the URL -or other conditions. The main features are: - - * Requests can be matched based on URL host, path, path prefix, schemes, - header and query values, HTTP methods or using custom matchers. - * URL hosts and paths can have variables with an optional regular - expression. - * Registered URLs can be built, or "reversed", which helps maintaining - references to resources. - * Routes can be used as subrouters: nested routes are only tested if the - parent route matches. This is useful to define groups of routes that - share common conditions like a host, a path prefix or other repeated - attributes. As a bonus, this optimizes request matching. - * It implements the http.Handler interface so it is compatible with the - standard http.ServeMux. - -Let's start registering a couple of URL paths and handlers: - - func main() { - r := mux.NewRouter() - r.HandleFunc("/", HomeHandler) - r.HandleFunc("/products", ProductsHandler) - r.HandleFunc("/articles", ArticlesHandler) - http.Handle("/", r) - } - -Here we register three routes mapping URL paths to handlers. This is -equivalent to how http.HandleFunc() works: if an incoming request URL matches -one of the paths, the corresponding handler is called passing -(http.ResponseWriter, *http.Request) as parameters. - -Paths can have variables. They are defined using the format {name} or -{name:pattern}. If a regular expression pattern is not defined, the matched -variable will be anything until the next slash. For example: - - r := mux.NewRouter() - r.HandleFunc("/products/{key}", ProductHandler) - r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler) - r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) - -The names are used to create a map of route variables which can be retrieved -calling mux.Vars(): - - vars := mux.Vars(request) - category := vars["category"] - -And this is all you need to know about the basic usage. More advanced options -are explained below. - -Routes can also be restricted to a domain or subdomain. Just define a host -pattern to be matched. They can also have variables: - - r := mux.NewRouter() - // Only matches if domain is "www.domain.com". - r.Host("www.domain.com") - // Matches a dynamic subdomain. - r.Host("{subdomain:[a-z]+}.domain.com") - -There are several other matchers that can be added. To match path prefixes: - - r.PathPrefix("/products/") - -...or HTTP methods: - - r.Methods("GET", "POST") - -...or URL schemes: - - r.Schemes("https") - -...or header values: - - r.Headers("X-Requested-With", "XMLHttpRequest") - -...or query values: - - r.Queries("key", "value") - -...or to use a custom matcher function: - - r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool { - return r.ProtoMajor == 0 - }) - -...and finally, it is possible to combine several matchers in a single route: - - r.HandleFunc("/products", ProductsHandler). - Host("www.domain.com"). - Methods("GET"). - Schemes("http") - -Setting the same matching conditions again and again can be boring, so we have -a way to group several routes that share the same requirements. -We call it "subrouting". - -For example, let's say we have several URLs that should only match when the -host is "www.domain.com". Create a route for that host and get a "subrouter" -from it: - - r := mux.NewRouter() - s := r.Host("www.domain.com").Subrouter() - -Then register routes in the subrouter: - - s.HandleFunc("/products/", ProductsHandler) - s.HandleFunc("/products/{key}", ProductHandler) - s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) - -The three URL paths we registered above will only be tested if the domain is -"www.domain.com", because the subrouter is tested first. This is not -only convenient, but also optimizes request matching. You can create -subrouters combining any attribute matchers accepted by a route. - -Subrouters can be used to create domain or path "namespaces": you define -subrouters in a central place and then parts of the app can register its -paths relatively to a given subrouter. - -There's one more thing about subroutes. When a subrouter has a path prefix, -the inner routes use it as base for their paths: - - r := mux.NewRouter() - s := r.PathPrefix("/products").Subrouter() - // "/products/" - s.HandleFunc("/", ProductsHandler) - // "/products/{key}/" - s.HandleFunc("/{key}/", ProductHandler) - // "/products/{key}/details" - s.HandleFunc("/{key}/details", ProductDetailsHandler) - -Now let's see how to build registered URLs. - -Routes can be named. All routes that define a name can have their URLs built, -or "reversed". We define a name calling Name() on a route. For example: - - r := mux.NewRouter() - r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). - Name("article") - -To build a URL, get the route and call the URL() method, passing a sequence of -key/value pairs for the route variables. For the previous route, we would do: - - url, err := r.Get("article").URL("category", "technology", "id", "42") - -...and the result will be a url.URL with the following path: - - "/articles/technology/42" - -This also works for host variables: - - r := mux.NewRouter() - r.Host("{subdomain}.domain.com"). - Path("/articles/{category}/{id:[0-9]+}"). - HandlerFunc(ArticleHandler). - Name("article") - - // url.String() will be "http://news.domain.com/articles/technology/42" - url, err := r.Get("article").URL("subdomain", "news", - "category", "technology", - "id", "42") - -All variables defined in the route are required, and their values must -conform to the corresponding patterns. These requirements guarantee that a -generated URL will always match a registered route -- the only exception is -for explicitly defined "build-only" routes which never match. - -There's also a way to build only the URL host or path for a route: -use the methods URLHost() or URLPath() instead. For the previous route, -we would do: - - // "http://news.domain.com/" - host, err := r.Get("article").URLHost("subdomain", "news") - - // "/articles/technology/42" - path, err := r.Get("article").URLPath("category", "technology", "id", "42") - -And if you use subrouters, host and path defined separately can be built -as well: - - r := mux.NewRouter() - s := r.Host("{subdomain}.domain.com").Subrouter() - s.Path("/articles/{category}/{id:[0-9]+}"). - HandlerFunc(ArticleHandler). - Name("article") - - // "http://news.domain.com/articles/technology/42" - url, err := r.Get("article").URL("subdomain", "news", - "category", "technology", - "id", "42") -*/ -package mux diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/mux.go b/Godeps/_workspace/src/github.com/gorilla/mux/mux.go deleted file mode 100644 index 26966422b..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/mux.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "fmt" - "net/http" - "path" - - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/context" -) - -// NewRouter returns a new router instance. -func NewRouter() *Router { - return &Router{namedRoutes: make(map[string]*Route), KeepContext: false} -} - -// Router registers routes to be matched and dispatches a handler. -// -// It implements the http.Handler interface, so it can be registered to serve -// requests: -// -// var router = mux.NewRouter() -// -// func main() { -// http.Handle("/", router) -// } -// -// Or, for Google App Engine, register it in a init() function: -// -// func init() { -// http.Handle("/", router) -// } -// -// This will send all incoming requests to the router. -type Router struct { - // Configurable Handler to be used when no route matches. - NotFoundHandler http.Handler - // Parent route, if this is a subrouter. - parent parentRoute - // Routes to be matched, in order. - routes []*Route - // Routes by name for URL building. - namedRoutes map[string]*Route - // See Router.StrictSlash(). This defines the flag for new routes. - strictSlash bool - // If true, do not clear the request context after handling the request - KeepContext bool -} - -// Match matches registered routes against the request. -func (r *Router) Match(req *http.Request, match *RouteMatch) bool { - for _, route := range r.routes { - if route.Match(req, match) { - return true - } - } - return false -} - -// ServeHTTP dispatches the handler registered in the matched route. -// -// When there is a match, the route variables can be retrieved calling -// mux.Vars(request). -func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // Clean path to canonical form and redirect. - if p := cleanPath(req.URL.Path); p != req.URL.Path { - - // Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query. - // This matches with fix in go 1.2 r.c. 4 for same problem. Go Issue: - // http://code.google.com/p/go/issues/detail?id=5252 - url := *req.URL - url.Path = p - p = url.String() - - w.Header().Set("Location", p) - w.WriteHeader(http.StatusMovedPermanently) - return - } - var match RouteMatch - var handler http.Handler - if r.Match(req, &match) { - handler = match.Handler - setVars(req, match.Vars) - setCurrentRoute(req, match.Route) - } - if handler == nil { - handler = r.NotFoundHandler - if handler == nil { - handler = http.NotFoundHandler() - } - } - if !r.KeepContext { - defer context.Clear(req) - } - handler.ServeHTTP(w, req) -} - -// Get returns a route registered with the given name. -func (r *Router) Get(name string) *Route { - return r.getNamedRoutes()[name] -} - -// GetRoute returns a route registered with the given name. This method -// was renamed to Get() and remains here for backwards compatibility. -func (r *Router) GetRoute(name string) *Route { - return r.getNamedRoutes()[name] -} - -// StrictSlash defines the trailing slash behavior for new routes. The initial -// value is false. -// -// When true, if the route path is "/path/", accessing "/path" will redirect -// to the former and vice versa. In other words, your application will always -// see the path as specified in the route. -// -// When false, if the route path is "/path", accessing "/path/" will not match -// this route and vice versa. -// -// Special case: when a route sets a path prefix using the PathPrefix() method, -// strict slash is ignored for that route because the redirect behavior can't -// be determined from a prefix alone. However, any subrouters created from that -// route inherit the original StrictSlash setting. -func (r *Router) StrictSlash(value bool) *Router { - r.strictSlash = value - return r -} - -// ---------------------------------------------------------------------------- -// parentRoute -// ---------------------------------------------------------------------------- - -// getNamedRoutes returns the map where named routes are registered. -func (r *Router) getNamedRoutes() map[string]*Route { - if r.namedRoutes == nil { - if r.parent != nil { - r.namedRoutes = r.parent.getNamedRoutes() - } else { - r.namedRoutes = make(map[string]*Route) - } - } - return r.namedRoutes -} - -// getRegexpGroup returns regexp definitions from the parent route, if any. -func (r *Router) getRegexpGroup() *routeRegexpGroup { - if r.parent != nil { - return r.parent.getRegexpGroup() - } - return nil -} - -// ---------------------------------------------------------------------------- -// Route factories -// ---------------------------------------------------------------------------- - -// NewRoute registers an empty route. -func (r *Router) NewRoute() *Route { - route := &Route{parent: r, strictSlash: r.strictSlash} - r.routes = append(r.routes, route) - return route -} - -// Handle registers a new route with a matcher for the URL path. -// See Route.Path() and Route.Handler(). -func (r *Router) Handle(path string, handler http.Handler) *Route { - return r.NewRoute().Path(path).Handler(handler) -} - -// HandleFunc registers a new route with a matcher for the URL path. -// See Route.Path() and Route.HandlerFunc(). -func (r *Router) HandleFunc(path string, f func(http.ResponseWriter, - *http.Request)) *Route { - return r.NewRoute().Path(path).HandlerFunc(f) -} - -// Headers registers a new route with a matcher for request header values. -// See Route.Headers(). -func (r *Router) Headers(pairs ...string) *Route { - return r.NewRoute().Headers(pairs...) -} - -// Host registers a new route with a matcher for the URL host. -// See Route.Host(). -func (r *Router) Host(tpl string) *Route { - return r.NewRoute().Host(tpl) -} - -// MatcherFunc registers a new route with a custom matcher function. -// See Route.MatcherFunc(). -func (r *Router) MatcherFunc(f MatcherFunc) *Route { - return r.NewRoute().MatcherFunc(f) -} - -// Methods registers a new route with a matcher for HTTP methods. -// See Route.Methods(). -func (r *Router) Methods(methods ...string) *Route { - return r.NewRoute().Methods(methods...) -} - -// Path registers a new route with a matcher for the URL path. -// See Route.Path(). -func (r *Router) Path(tpl string) *Route { - return r.NewRoute().Path(tpl) -} - -// PathPrefix registers a new route with a matcher for the URL path prefix. -// See Route.PathPrefix(). -func (r *Router) PathPrefix(tpl string) *Route { - return r.NewRoute().PathPrefix(tpl) -} - -// Queries registers a new route with a matcher for URL query values. -// See Route.Queries(). -func (r *Router) Queries(pairs ...string) *Route { - return r.NewRoute().Queries(pairs...) -} - -// Schemes registers a new route with a matcher for URL schemes. -// See Route.Schemes(). -func (r *Router) Schemes(schemes ...string) *Route { - return r.NewRoute().Schemes(schemes...) -} - -// ---------------------------------------------------------------------------- -// Context -// ---------------------------------------------------------------------------- - -// RouteMatch stores information about a matched route. -type RouteMatch struct { - Route *Route - Handler http.Handler - Vars map[string]string -} - -type contextKey int - -const ( - varsKey contextKey = iota - routeKey -) - -// Vars returns the route variables for the current request, if any. -func Vars(r *http.Request) map[string]string { - if rv := context.Get(r, varsKey); rv != nil { - return rv.(map[string]string) - } - return nil -} - -// CurrentRoute returns the matched route for the current request, if any. -func CurrentRoute(r *http.Request) *Route { - if rv := context.Get(r, routeKey); rv != nil { - return rv.(*Route) - } - return nil -} - -func setVars(r *http.Request, val interface{}) { - context.Set(r, varsKey, val) -} - -func setCurrentRoute(r *http.Request, val interface{}) { - context.Set(r, routeKey, val) -} - -// ---------------------------------------------------------------------------- -// Helpers -// ---------------------------------------------------------------------------- - -// cleanPath returns the canonical path for p, eliminating . and .. elements. -// Borrowed from the net/http package. -func cleanPath(p string) string { - if p == "" { - return "/" - } - if p[0] != '/' { - p = "/" + p - } - np := path.Clean(p) - // path.Clean removes trailing slash except for root; - // put the trailing slash back if necessary. - if p[len(p)-1] == '/' && np != "/" { - np += "/" - } - return np -} - -// uniqueVars returns an error if two slices contain duplicated strings. -func uniqueVars(s1, s2 []string) error { - for _, v1 := range s1 { - for _, v2 := range s2 { - if v1 == v2 { - return fmt.Errorf("mux: duplicated route variable %q", v2) - } - } - } - return nil -} - -// mapFromPairs converts variadic string parameters to a string map. -func mapFromPairs(pairs ...string) (map[string]string, error) { - length := len(pairs) - if length%2 != 0 { - return nil, fmt.Errorf( - "mux: number of parameters must be multiple of 2, got %v", pairs) - } - m := make(map[string]string, length/2) - for i := 0; i < length; i += 2 { - m[pairs[i]] = pairs[i+1] - } - return m, nil -} - -// matchInArray returns true if the given string value is in the array. -func matchInArray(arr []string, value string) bool { - for _, v := range arr { - if v == value { - return true - } - } - return false -} - -// matchMap returns true if the given key/value pairs exist in a given map. -func matchMap(toCheck map[string]string, toMatch map[string][]string, - canonicalKey bool) bool { - for k, v := range toCheck { - // Check if key exists. - if canonicalKey { - k = http.CanonicalHeaderKey(k) - } - if values := toMatch[k]; values == nil { - return false - } else if v != "" { - // If value was defined as an empty string we only check that the - // key exists. Otherwise we also check for equality. - valueExists := false - for _, value := range values { - if v == value { - valueExists = true - break - } - } - if !valueExists { - return false - } - } - } - return true -} diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/mux_test.go b/Godeps/_workspace/src/github.com/gorilla/mux/mux_test.go deleted file mode 100644 index 4c2706b3c..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/mux_test.go +++ /dev/null @@ -1,943 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "fmt" - "net/http" - "testing" - - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/context" -) - -type routeTest struct { - title string // title of the test - route *Route // the route being tested - request *http.Request // a request to test the route - vars map[string]string // the expected vars of the match - host string // the expected host of the match - path string // the expected path of the match - shouldMatch bool // whether the request is expected to match the route at all - shouldRedirect bool // whether the request should result in a redirect -} - -func TestHost(t *testing.T) { - // newRequestHost a new request with a method, url, and host header - newRequestHost := func(method, url, host string) *http.Request { - req, err := http.NewRequest(method, url, nil) - if err != nil { - panic(err) - } - req.Host = host - return req - } - - tests := []routeTest{ - { - title: "Host route match", - route: new(Route).Host("aaa.bbb.ccc"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: true, - }, - { - title: "Host route, wrong host in request URL", - route: new(Route).Host("aaa.bbb.ccc"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: false, - }, - { - title: "Host route with port, match", - route: new(Route).Host("aaa.bbb.ccc:1234"), - request: newRequest("GET", "http://aaa.bbb.ccc:1234/111/222/333"), - vars: map[string]string{}, - host: "aaa.bbb.ccc:1234", - path: "", - shouldMatch: true, - }, - { - title: "Host route with port, wrong port in request URL", - route: new(Route).Host("aaa.bbb.ccc:1234"), - request: newRequest("GET", "http://aaa.bbb.ccc:9999/111/222/333"), - vars: map[string]string{}, - host: "aaa.bbb.ccc:1234", - path: "", - shouldMatch: false, - }, - { - title: "Host route, match with host in request header", - route: new(Route).Host("aaa.bbb.ccc"), - request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc"), - vars: map[string]string{}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: true, - }, - { - title: "Host route, wrong host in request header", - route: new(Route).Host("aaa.bbb.ccc"), - request: newRequestHost("GET", "/111/222/333", "aaa.222.ccc"), - vars: map[string]string{}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: false, - }, - // BUG {new(Route).Host("aaa.bbb.ccc:1234"), newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:1234"), map[string]string{}, "aaa.bbb.ccc:1234", "", true}, - { - title: "Host route with port, wrong host in request header", - route: new(Route).Host("aaa.bbb.ccc:1234"), - request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:9999"), - vars: map[string]string{}, - host: "aaa.bbb.ccc:1234", - path: "", - shouldMatch: false, - }, - { - title: "Host route with pattern, match", - route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{"v1": "bbb"}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: true, - }, - { - title: "Host route with pattern, wrong host in request URL", - route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{"v1": "bbb"}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: false, - }, - { - title: "Host route with multiple patterns, match", - route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: true, - }, - { - title: "Host route with multiple patterns, wrong host in request URL", - route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"}, - host: "aaa.bbb.ccc", - path: "", - shouldMatch: false, - }, - } - for _, test := range tests { - testRoute(t, test) - } -} - -func TestPath(t *testing.T) { - tests := []routeTest{ - { - title: "Path route, match", - route: new(Route).Path("/111/222/333"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{}, - host: "", - path: "/111/222/333", - shouldMatch: true, - }, - { - title: "Path route, match with trailing slash in request and path", - route: new(Route).Path("/111/"), - request: newRequest("GET", "http://localhost/111/"), - vars: map[string]string{}, - host: "", - path: "/111/", - shouldMatch: true, - }, - { - title: "Path route, do not match with trailing slash in path", - route: new(Route).Path("/111/"), - request: newRequest("GET", "http://localhost/111"), - vars: map[string]string{}, - host: "", - path: "/111", - shouldMatch: false, - }, - { - title: "Path route, do not match with trailing slash in request", - route: new(Route).Path("/111"), - request: newRequest("GET", "http://localhost/111/"), - vars: map[string]string{}, - host: "", - path: "/111/", - shouldMatch: false, - }, - { - title: "Path route, wrong path in request in request URL", - route: new(Route).Path("/111/222/333"), - request: newRequest("GET", "http://localhost/1/2/3"), - vars: map[string]string{}, - host: "", - path: "/111/222/333", - shouldMatch: false, - }, - { - title: "Path route with pattern, match", - route: new(Route).Path("/111/{v1:[0-9]{3}}/333"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{"v1": "222"}, - host: "", - path: "/111/222/333", - shouldMatch: true, - }, - { - title: "Path route with pattern, URL in request does not match", - route: new(Route).Path("/111/{v1:[0-9]{3}}/333"), - request: newRequest("GET", "http://localhost/111/aaa/333"), - vars: map[string]string{"v1": "222"}, - host: "", - path: "/111/222/333", - shouldMatch: false, - }, - { - title: "Path route with multiple patterns, match", - route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"}, - host: "", - path: "/111/222/333", - shouldMatch: true, - }, - { - title: "Path route with multiple patterns, URL in request does not match", - route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/aaa/333"), - vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"}, - host: "", - path: "/111/222/333", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestPathPrefix(t *testing.T) { - tests := []routeTest{ - { - title: "PathPrefix route, match", - route: new(Route).PathPrefix("/111"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{}, - host: "", - path: "/111", - shouldMatch: true, - }, - { - title: "PathPrefix route, match substring", - route: new(Route).PathPrefix("/1"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{}, - host: "", - path: "/1", - shouldMatch: true, - }, - { - title: "PathPrefix route, URL prefix in request does not match", - route: new(Route).PathPrefix("/111"), - request: newRequest("GET", "http://localhost/1/2/3"), - vars: map[string]string{}, - host: "", - path: "/111", - shouldMatch: false, - }, - { - title: "PathPrefix route with pattern, match", - route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{"v1": "222"}, - host: "", - path: "/111/222", - shouldMatch: true, - }, - { - title: "PathPrefix route with pattern, URL prefix in request does not match", - route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/aaa/333"), - vars: map[string]string{"v1": "222"}, - host: "", - path: "/111/222", - shouldMatch: false, - }, - { - title: "PathPrefix route with multiple patterns, match", - route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/222/333"), - vars: map[string]string{"v1": "111", "v2": "222"}, - host: "", - path: "/111/222", - shouldMatch: true, - }, - { - title: "PathPrefix route with multiple patterns, URL prefix in request does not match", - route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"), - request: newRequest("GET", "http://localhost/111/aaa/333"), - vars: map[string]string{"v1": "111", "v2": "222"}, - host: "", - path: "/111/222", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestHostPath(t *testing.T) { - tests := []routeTest{ - { - title: "Host and Path route, match", - route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Host and Path route, wrong host in request URL", - route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - { - title: "Host and Path route with pattern, match", - route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{"v1": "bbb", "v2": "222"}, - host: "aaa.bbb.ccc", - path: "/111/222/333", - shouldMatch: true, - }, - { - title: "Host and Path route with pattern, URL in request does not match", - route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{"v1": "bbb", "v2": "222"}, - host: "aaa.bbb.ccc", - path: "/111/222/333", - shouldMatch: false, - }, - { - title: "Host and Path route with multiple patterns, match", - route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"), - request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), - vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"}, - host: "aaa.bbb.ccc", - path: "/111/222/333", - shouldMatch: true, - }, - { - title: "Host and Path route with multiple patterns, URL in request does not match", - route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"), - request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), - vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"}, - host: "aaa.bbb.ccc", - path: "/111/222/333", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestHeaders(t *testing.T) { - // newRequestHeaders creates a new request with a method, url, and headers - newRequestHeaders := func(method, url string, headers map[string]string) *http.Request { - req, err := http.NewRequest(method, url, nil) - if err != nil { - panic(err) - } - for k, v := range headers { - req.Header.Add(k, v) - } - return req - } - - tests := []routeTest{ - { - title: "Headers route, match", - route: new(Route).Headers("foo", "bar", "baz", "ding"), - request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "ding"}), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Headers route, bad header values", - route: new(Route).Headers("foo", "bar", "baz", "ding"), - request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "dong"}), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } - -} - -func TestMethods(t *testing.T) { - tests := []routeTest{ - { - title: "Methods route, match GET", - route: new(Route).Methods("GET", "POST"), - request: newRequest("GET", "http://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Methods route, match POST", - route: new(Route).Methods("GET", "POST"), - request: newRequest("POST", "http://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Methods route, bad method", - route: new(Route).Methods("GET", "POST"), - request: newRequest("PUT", "http://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestQueries(t *testing.T) { - tests := []routeTest{ - { - title: "Queries route, match", - route: new(Route).Queries("foo", "bar", "baz", "ding"), - request: newRequest("GET", "http://localhost?foo=bar&baz=ding"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route, match with a query string", - route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"), - request: newRequest("GET", "http://www.example.com/api?foo=bar&baz=ding"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route, match with a query string out of order", - route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"), - request: newRequest("GET", "http://www.example.com/api?baz=ding&foo=bar"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route, bad query", - route: new(Route).Queries("foo", "bar", "baz", "ding"), - request: newRequest("GET", "http://localhost?foo=bar&baz=dong"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - { - title: "Queries route with pattern, match", - route: new(Route).Queries("foo", "{v1}"), - request: newRequest("GET", "http://localhost?foo=bar"), - vars: map[string]string{"v1": "bar"}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route with multiple patterns, match", - route: new(Route).Queries("foo", "{v1}", "baz", "{v2}"), - request: newRequest("GET", "http://localhost?foo=bar&baz=ding"), - vars: map[string]string{"v1": "bar", "v2": "ding"}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route with regexp pattern, match", - route: new(Route).Queries("foo", "{v1:[0-9]+}"), - request: newRequest("GET", "http://localhost?foo=10"), - vars: map[string]string{"v1": "10"}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Queries route with regexp pattern, regexp does not match", - route: new(Route).Queries("foo", "{v1:[0-9]+}"), - request: newRequest("GET", "http://localhost?foo=a"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestSchemes(t *testing.T) { - tests := []routeTest{ - // Schemes - { - title: "Schemes route, match https", - route: new(Route).Schemes("https", "ftp"), - request: newRequest("GET", "https://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Schemes route, match ftp", - route: new(Route).Schemes("https", "ftp"), - request: newRequest("GET", "ftp://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "Schemes route, bad scheme", - route: new(Route).Schemes("https", "ftp"), - request: newRequest("GET", "http://localhost"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - } - for _, test := range tests { - testRoute(t, test) - } -} - -func TestMatcherFunc(t *testing.T) { - m := func(r *http.Request, m *RouteMatch) bool { - if r.URL.Host == "aaa.bbb.ccc" { - return true - } - return false - } - - tests := []routeTest{ - { - title: "MatchFunc route, match", - route: new(Route).MatcherFunc(m), - request: newRequest("GET", "http://aaa.bbb.ccc"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: true, - }, - { - title: "MatchFunc route, non-match", - route: new(Route).MatcherFunc(m), - request: newRequest("GET", "http://aaa.222.ccc"), - vars: map[string]string{}, - host: "", - path: "", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestSubRouter(t *testing.T) { - subrouter1 := new(Route).Host("{v1:[a-z]+}.google.com").Subrouter() - subrouter2 := new(Route).PathPrefix("/foo/{v1}").Subrouter() - - tests := []routeTest{ - { - route: subrouter1.Path("/{v2:[a-z]+}"), - request: newRequest("GET", "http://aaa.google.com/bbb"), - vars: map[string]string{"v1": "aaa", "v2": "bbb"}, - host: "aaa.google.com", - path: "/bbb", - shouldMatch: true, - }, - { - route: subrouter1.Path("/{v2:[a-z]+}"), - request: newRequest("GET", "http://111.google.com/111"), - vars: map[string]string{"v1": "aaa", "v2": "bbb"}, - host: "aaa.google.com", - path: "/bbb", - shouldMatch: false, - }, - { - route: subrouter2.Path("/baz/{v2}"), - request: newRequest("GET", "http://localhost/foo/bar/baz/ding"), - vars: map[string]string{"v1": "bar", "v2": "ding"}, - host: "", - path: "/foo/bar/baz/ding", - shouldMatch: true, - }, - { - route: subrouter2.Path("/baz/{v2}"), - request: newRequest("GET", "http://localhost/foo/bar"), - vars: map[string]string{"v1": "bar", "v2": "ding"}, - host: "", - path: "/foo/bar/baz/ding", - shouldMatch: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -func TestNamedRoutes(t *testing.T) { - r1 := NewRouter() - r1.NewRoute().Name("a") - r1.NewRoute().Name("b") - r1.NewRoute().Name("c") - - r2 := r1.NewRoute().Subrouter() - r2.NewRoute().Name("d") - r2.NewRoute().Name("e") - r2.NewRoute().Name("f") - - r3 := r2.NewRoute().Subrouter() - r3.NewRoute().Name("g") - r3.NewRoute().Name("h") - r3.NewRoute().Name("i") - - if r1.namedRoutes == nil || len(r1.namedRoutes) != 9 { - t.Errorf("Expected 9 named routes, got %v", r1.namedRoutes) - } else if r1.Get("i") == nil { - t.Errorf("Subroute name not registered") - } -} - -func TestStrictSlash(t *testing.T) { - r := NewRouter() - r.StrictSlash(true) - - tests := []routeTest{ - { - title: "Redirect path without slash", - route: r.NewRoute().Path("/111/"), - request: newRequest("GET", "http://localhost/111"), - vars: map[string]string{}, - host: "", - path: "/111/", - shouldMatch: true, - shouldRedirect: true, - }, - { - title: "Do not redirect path with slash", - route: r.NewRoute().Path("/111/"), - request: newRequest("GET", "http://localhost/111/"), - vars: map[string]string{}, - host: "", - path: "/111/", - shouldMatch: true, - shouldRedirect: false, - }, - { - title: "Redirect path with slash", - route: r.NewRoute().Path("/111"), - request: newRequest("GET", "http://localhost/111/"), - vars: map[string]string{}, - host: "", - path: "/111", - shouldMatch: true, - shouldRedirect: true, - }, - { - title: "Do not redirect path without slash", - route: r.NewRoute().Path("/111"), - request: newRequest("GET", "http://localhost/111"), - vars: map[string]string{}, - host: "", - path: "/111", - shouldMatch: true, - shouldRedirect: false, - }, - { - title: "Propagate StrictSlash to subrouters", - route: r.NewRoute().PathPrefix("/static/").Subrouter().Path("/images/"), - request: newRequest("GET", "http://localhost/static/images"), - vars: map[string]string{}, - host: "", - path: "/static/images/", - shouldMatch: true, - shouldRedirect: true, - }, - { - title: "Ignore StrictSlash for path prefix", - route: r.NewRoute().PathPrefix("/static/"), - request: newRequest("GET", "http://localhost/static/logo.png"), - vars: map[string]string{}, - host: "", - path: "/static/", - shouldMatch: true, - shouldRedirect: false, - }, - } - - for _, test := range tests { - testRoute(t, test) - } -} - -// ---------------------------------------------------------------------------- -// Helpers -// ---------------------------------------------------------------------------- - -func getRouteTemplate(route *Route) string { - host, path := "none", "none" - if route.regexp != nil { - if route.regexp.host != nil { - host = route.regexp.host.template - } - if route.regexp.path != nil { - path = route.regexp.path.template - } - } - return fmt.Sprintf("Host: %v, Path: %v", host, path) -} - -func testRoute(t *testing.T, test routeTest) { - request := test.request - route := test.route - vars := test.vars - shouldMatch := test.shouldMatch - host := test.host - path := test.path - url := test.host + test.path - shouldRedirect := test.shouldRedirect - - var match RouteMatch - ok := route.Match(request, &match) - if ok != shouldMatch { - msg := "Should match" - if !shouldMatch { - msg = "Should not match" - } - t.Errorf("(%v) %v:\nRoute: %#v\nRequest: %#v\nVars: %v\n", test.title, msg, route, request, vars) - return - } - if shouldMatch { - if test.vars != nil && !stringMapEqual(test.vars, match.Vars) { - t.Errorf("(%v) Vars not equal: expected %v, got %v", test.title, vars, match.Vars) - return - } - if host != "" { - u, _ := test.route.URLHost(mapToPairs(match.Vars)...) - if host != u.Host { - t.Errorf("(%v) URLHost not equal: expected %v, got %v -- %v", test.title, host, u.Host, getRouteTemplate(route)) - return - } - } - if path != "" { - u, _ := route.URLPath(mapToPairs(match.Vars)...) - if path != u.Path { - t.Errorf("(%v) URLPath not equal: expected %v, got %v -- %v", test.title, path, u.Path, getRouteTemplate(route)) - return - } - } - if url != "" { - u, _ := route.URL(mapToPairs(match.Vars)...) - if url != u.Host+u.Path { - t.Errorf("(%v) URL not equal: expected %v, got %v -- %v", test.title, url, u.Host+u.Path, getRouteTemplate(route)) - return - } - } - if shouldRedirect && match.Handler == nil { - t.Errorf("(%v) Did not redirect", test.title) - return - } - if !shouldRedirect && match.Handler != nil { - t.Errorf("(%v) Unexpected redirect", test.title) - return - } - } -} - -// Tests that the context is cleared or not cleared properly depending on -// the configuration of the router -func TestKeepContext(t *testing.T) { - func1 := func(w http.ResponseWriter, r *http.Request) {} - - r := NewRouter() - r.HandleFunc("/", func1).Name("func1") - - req, _ := http.NewRequest("GET", "http://localhost/", nil) - context.Set(req, "t", 1) - - res := new(http.ResponseWriter) - r.ServeHTTP(*res, req) - - if _, ok := context.GetOk(req, "t"); ok { - t.Error("Context should have been cleared at end of request") - } - - r.KeepContext = true - - req, _ = http.NewRequest("GET", "http://localhost/", nil) - context.Set(req, "t", 1) - - r.ServeHTTP(*res, req) - if _, ok := context.GetOk(req, "t"); !ok { - t.Error("Context should NOT have been cleared at end of request") - } - -} - -type TestA301ResponseWriter struct { - hh http.Header - status int -} - -func (ho TestA301ResponseWriter) Header() http.Header { - return http.Header(ho.hh) -} - -func (ho TestA301ResponseWriter) Write(b []byte) (int, error) { - return 0, nil -} - -func (ho TestA301ResponseWriter) WriteHeader(code int) { - ho.status = code -} - -func Test301Redirect(t *testing.T) { - m := make(http.Header) - - func1 := func(w http.ResponseWriter, r *http.Request) {} - func2 := func(w http.ResponseWriter, r *http.Request) {} - - r := NewRouter() - r.HandleFunc("/api/", func2).Name("func2") - r.HandleFunc("/", func1).Name("func1") - - req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil) - - res := TestA301ResponseWriter{ - hh: m, - status: 0, - } - r.ServeHTTP(&res, req) - - if "http://localhost/api/?abc=def" != res.hh["Location"][0] { - t.Errorf("Should have complete URL with query string") - } -} - -// https://plus.google.com/101022900381697718949/posts/eWy6DjFJ6uW -func TestSubrouterHeader(t *testing.T) { - expected := "func1 response" - func1 := func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, expected) - } - func2 := func(http.ResponseWriter, *http.Request) {} - - r := NewRouter() - s := r.Headers("SomeSpecialHeader", "").Subrouter() - s.HandleFunc("/", func1).Name("func1") - r.HandleFunc("/", func2).Name("func2") - - req, _ := http.NewRequest("GET", "http://localhost/", nil) - req.Header.Add("SomeSpecialHeader", "foo") - match := new(RouteMatch) - matched := r.Match(req, match) - if !matched { - t.Errorf("Should match request") - } - if match.Route.GetName() != "func1" { - t.Errorf("Expecting func1 handler, got %s", match.Route.GetName()) - } - resp := NewRecorder() - match.Handler.ServeHTTP(resp, req) - if resp.Body.String() != expected { - t.Errorf("Expecting %q", expected) - } -} - -// mapToPairs converts a string map to a slice of string pairs -func mapToPairs(m map[string]string) []string { - var i int - p := make([]string, len(m)*2) - for k, v := range m { - p[i] = k - p[i+1] = v - i += 2 - } - return p -} - -// stringMapEqual checks the equality of two string maps -func stringMapEqual(m1, m2 map[string]string) bool { - nil1 := m1 == nil - nil2 := m2 == nil - if nil1 != nil2 || len(m1) != len(m2) { - return false - } - for k, v := range m1 { - if v != m2[k] { - return false - } - } - return true -} - -// newRequest is a helper function to create a new request with a method and url -func newRequest(method, url string) *http.Request { - req, err := http.NewRequest(method, url, nil) - if err != nil { - panic(err) - } - return req -} diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/old_test.go b/Godeps/_workspace/src/github.com/gorilla/mux/old_test.go deleted file mode 100644 index 1f7c190c0..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/old_test.go +++ /dev/null @@ -1,714 +0,0 @@ -// Old tests ported to Go1. This is a mess. Want to drop it one day. - -// Copyright 2011 Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "bytes" - "net/http" - "testing" -) - -// ---------------------------------------------------------------------------- -// ResponseRecorder -// ---------------------------------------------------------------------------- -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// ResponseRecorder is an implementation of http.ResponseWriter that -// records its mutations for later inspection in tests. -type ResponseRecorder struct { - Code int // the HTTP response code from WriteHeader - HeaderMap http.Header // the HTTP response headers - Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to - Flushed bool -} - -// NewRecorder returns an initialized ResponseRecorder. -func NewRecorder() *ResponseRecorder { - return &ResponseRecorder{ - HeaderMap: make(http.Header), - Body: new(bytes.Buffer), - } -} - -// DefaultRemoteAddr is the default remote address to return in RemoteAddr if -// an explicit DefaultRemoteAddr isn't set on ResponseRecorder. -const DefaultRemoteAddr = "1.2.3.4" - -// Header returns the response headers. -func (rw *ResponseRecorder) Header() http.Header { - return rw.HeaderMap -} - -// Write always succeeds and writes to rw.Body, if not nil. -func (rw *ResponseRecorder) Write(buf []byte) (int, error) { - if rw.Body != nil { - rw.Body.Write(buf) - } - if rw.Code == 0 { - rw.Code = http.StatusOK - } - return len(buf), nil -} - -// WriteHeader sets rw.Code. -func (rw *ResponseRecorder) WriteHeader(code int) { - rw.Code = code -} - -// Flush sets rw.Flushed to true. -func (rw *ResponseRecorder) Flush() { - rw.Flushed = true -} - -// ---------------------------------------------------------------------------- - -func TestRouteMatchers(t *testing.T) { - var scheme, host, path, query, method string - var headers map[string]string - var resultVars map[bool]map[string]string - - router := NewRouter() - router.NewRoute().Host("{var1}.google.com"). - Path("/{var2:[a-z]+}/{var3:[0-9]+}"). - Queries("foo", "bar"). - Methods("GET"). - Schemes("https"). - Headers("x-requested-with", "XMLHttpRequest") - router.NewRoute().Host("www.{var4}.com"). - PathPrefix("/foo/{var5:[a-z]+}/{var6:[0-9]+}"). - Queries("baz", "ding"). - Methods("POST"). - Schemes("http"). - Headers("Content-Type", "application/json") - - reset := func() { - // Everything match. - scheme = "https" - host = "www.google.com" - path = "/product/42" - query = "?foo=bar" - method = "GET" - headers = map[string]string{"X-Requested-With": "XMLHttpRequest"} - resultVars = map[bool]map[string]string{ - true: {"var1": "www", "var2": "product", "var3": "42"}, - false: {}, - } - } - - reset2 := func() { - // Everything match. - scheme = "http" - host = "www.google.com" - path = "/foo/product/42/path/that/is/ignored" - query = "?baz=ding" - method = "POST" - headers = map[string]string{"Content-Type": "application/json"} - resultVars = map[bool]map[string]string{ - true: {"var4": "google", "var5": "product", "var6": "42"}, - false: {}, - } - } - - match := func(shouldMatch bool) { - url := scheme + "://" + host + path + query - request, _ := http.NewRequest(method, url, nil) - for key, value := range headers { - request.Header.Add(key, value) - } - - var routeMatch RouteMatch - matched := router.Match(request, &routeMatch) - if matched != shouldMatch { - // Need better messages. :) - if matched { - t.Errorf("Should match.") - } else { - t.Errorf("Should not match.") - } - } - - if matched { - currentRoute := routeMatch.Route - if currentRoute == nil { - t.Errorf("Expected a current route.") - } - vars := routeMatch.Vars - expectedVars := resultVars[shouldMatch] - if len(vars) != len(expectedVars) { - t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars) - } - for name, value := range vars { - if expectedVars[name] != value { - t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars) - } - } - } - } - - // 1st route -------------------------------------------------------------- - - // Everything match. - reset() - match(true) - - // Scheme doesn't match. - reset() - scheme = "http" - match(false) - - // Host doesn't match. - reset() - host = "www.mygoogle.com" - match(false) - - // Path doesn't match. - reset() - path = "/product/notdigits" - match(false) - - // Query doesn't match. - reset() - query = "?foo=baz" - match(false) - - // Method doesn't match. - reset() - method = "POST" - match(false) - - // Header doesn't match. - reset() - headers = map[string]string{} - match(false) - - // Everything match, again. - reset() - match(true) - - // 2nd route -------------------------------------------------------------- - - // Everything match. - reset2() - match(true) - - // Scheme doesn't match. - reset2() - scheme = "https" - match(false) - - // Host doesn't match. - reset2() - host = "sub.google.com" - match(false) - - // Path doesn't match. - reset2() - path = "/bar/product/42" - match(false) - - // Query doesn't match. - reset2() - query = "?foo=baz" - match(false) - - // Method doesn't match. - reset2() - method = "GET" - match(false) - - // Header doesn't match. - reset2() - headers = map[string]string{} - match(false) - - // Everything match, again. - reset2() - match(true) -} - -type headerMatcherTest struct { - matcher headerMatcher - headers map[string]string - result bool -} - -var headerMatcherTests = []headerMatcherTest{ - { - matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}), - headers: map[string]string{"X-Requested-With": "XMLHttpRequest"}, - result: true, - }, - { - matcher: headerMatcher(map[string]string{"x-requested-with": ""}), - headers: map[string]string{"X-Requested-With": "anything"}, - result: true, - }, - { - matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}), - headers: map[string]string{}, - result: false, - }, -} - -type hostMatcherTest struct { - matcher *Route - url string - vars map[string]string - result bool -} - -var hostMatcherTests = []hostMatcherTest{ - { - matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"), - url: "http://abc.def.ghi/", - vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"}, - result: true, - }, - { - matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"), - url: "http://a.b.c/", - vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"}, - result: false, - }, -} - -type methodMatcherTest struct { - matcher methodMatcher - method string - result bool -} - -var methodMatcherTests = []methodMatcherTest{ - { - matcher: methodMatcher([]string{"GET", "POST", "PUT"}), - method: "GET", - result: true, - }, - { - matcher: methodMatcher([]string{"GET", "POST", "PUT"}), - method: "POST", - result: true, - }, - { - matcher: methodMatcher([]string{"GET", "POST", "PUT"}), - method: "PUT", - result: true, - }, - { - matcher: methodMatcher([]string{"GET", "POST", "PUT"}), - method: "DELETE", - result: false, - }, -} - -type pathMatcherTest struct { - matcher *Route - url string - vars map[string]string - result bool -} - -var pathMatcherTests = []pathMatcherTest{ - { - matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"), - url: "http://localhost:8080/123/456/789", - vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"}, - result: true, - }, - { - matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"), - url: "http://localhost:8080/1/2/3", - vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"}, - result: false, - }, -} - -type schemeMatcherTest struct { - matcher schemeMatcher - url string - result bool -} - -var schemeMatcherTests = []schemeMatcherTest{ - { - matcher: schemeMatcher([]string{"http", "https"}), - url: "http://localhost:8080/", - result: true, - }, - { - matcher: schemeMatcher([]string{"http", "https"}), - url: "https://localhost:8080/", - result: true, - }, - { - matcher: schemeMatcher([]string{"https"}), - url: "http://localhost:8080/", - result: false, - }, - { - matcher: schemeMatcher([]string{"http"}), - url: "https://localhost:8080/", - result: false, - }, -} - -type urlBuildingTest struct { - route *Route - vars []string - url string -} - -var urlBuildingTests = []urlBuildingTest{ - { - route: new(Route).Host("foo.domain.com"), - vars: []string{}, - url: "http://foo.domain.com", - }, - { - route: new(Route).Host("{subdomain}.domain.com"), - vars: []string{"subdomain", "bar"}, - url: "http://bar.domain.com", - }, - { - route: new(Route).Host("foo.domain.com").Path("/articles"), - vars: []string{}, - url: "http://foo.domain.com/articles", - }, - { - route: new(Route).Path("/articles"), - vars: []string{}, - url: "/articles", - }, - { - route: new(Route).Path("/articles/{category}/{id:[0-9]+}"), - vars: []string{"category", "technology", "id", "42"}, - url: "/articles/technology/42", - }, - { - route: new(Route).Host("{subdomain}.domain.com").Path("/articles/{category}/{id:[0-9]+}"), - vars: []string{"subdomain", "foo", "category", "technology", "id", "42"}, - url: "http://foo.domain.com/articles/technology/42", - }, -} - -func TestHeaderMatcher(t *testing.T) { - for _, v := range headerMatcherTests { - request, _ := http.NewRequest("GET", "http://localhost:8080/", nil) - for key, value := range v.headers { - request.Header.Add(key, value) - } - var routeMatch RouteMatch - result := v.matcher.Match(request, &routeMatch) - if result != v.result { - if v.result { - t.Errorf("%#v: should match %v.", v.matcher, request.Header) - } else { - t.Errorf("%#v: should not match %v.", v.matcher, request.Header) - } - } - } -} - -func TestHostMatcher(t *testing.T) { - for _, v := range hostMatcherTests { - request, _ := http.NewRequest("GET", v.url, nil) - var routeMatch RouteMatch - result := v.matcher.Match(request, &routeMatch) - vars := routeMatch.Vars - if result != v.result { - if v.result { - t.Errorf("%#v: should match %v.", v.matcher, v.url) - } else { - t.Errorf("%#v: should not match %v.", v.matcher, v.url) - } - } - if result { - if len(vars) != len(v.vars) { - t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars)) - } - for name, value := range vars { - if v.vars[name] != value { - t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value) - } - } - } else { - if len(vars) != 0 { - t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars)) - } - } - } -} - -func TestMethodMatcher(t *testing.T) { - for _, v := range methodMatcherTests { - request, _ := http.NewRequest(v.method, "http://localhost:8080/", nil) - var routeMatch RouteMatch - result := v.matcher.Match(request, &routeMatch) - if result != v.result { - if v.result { - t.Errorf("%#v: should match %v.", v.matcher, v.method) - } else { - t.Errorf("%#v: should not match %v.", v.matcher, v.method) - } - } - } -} - -func TestPathMatcher(t *testing.T) { - for _, v := range pathMatcherTests { - request, _ := http.NewRequest("GET", v.url, nil) - var routeMatch RouteMatch - result := v.matcher.Match(request, &routeMatch) - vars := routeMatch.Vars - if result != v.result { - if v.result { - t.Errorf("%#v: should match %v.", v.matcher, v.url) - } else { - t.Errorf("%#v: should not match %v.", v.matcher, v.url) - } - } - if result { - if len(vars) != len(v.vars) { - t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars)) - } - for name, value := range vars { - if v.vars[name] != value { - t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value) - } - } - } else { - if len(vars) != 0 { - t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars)) - } - } - } -} - -func TestSchemeMatcher(t *testing.T) { - for _, v := range schemeMatcherTests { - request, _ := http.NewRequest("GET", v.url, nil) - var routeMatch RouteMatch - result := v.matcher.Match(request, &routeMatch) - if result != v.result { - if v.result { - t.Errorf("%#v: should match %v.", v.matcher, v.url) - } else { - t.Errorf("%#v: should not match %v.", v.matcher, v.url) - } - } - } -} - -func TestUrlBuilding(t *testing.T) { - - for _, v := range urlBuildingTests { - u, _ := v.route.URL(v.vars...) - url := u.String() - if url != v.url { - t.Errorf("expected %v, got %v", v.url, url) - /* - reversePath := "" - reverseHost := "" - if v.route.pathTemplate != nil { - reversePath = v.route.pathTemplate.Reverse - } - if v.route.hostTemplate != nil { - reverseHost = v.route.hostTemplate.Reverse - } - - t.Errorf("%#v:\nexpected: %q\ngot: %q\nreverse path: %q\nreverse host: %q", v.route, v.url, url, reversePath, reverseHost) - */ - } - } - - ArticleHandler := func(w http.ResponseWriter, r *http.Request) { - } - - router := NewRouter() - router.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).Name("article") - - url, _ := router.Get("article").URL("category", "technology", "id", "42") - expected := "/articles/technology/42" - if url.String() != expected { - t.Errorf("Expected %v, got %v", expected, url.String()) - } -} - -func TestMatchedRouteName(t *testing.T) { - routeName := "stock" - router := NewRouter() - route := router.NewRoute().Path("/products/").Name(routeName) - - url := "http://www.domain.com/products/" - request, _ := http.NewRequest("GET", url, nil) - var rv RouteMatch - ok := router.Match(request, &rv) - - if !ok || rv.Route != route { - t.Errorf("Expected same route, got %+v.", rv.Route) - } - - retName := rv.Route.GetName() - if retName != routeName { - t.Errorf("Expected %q, got %q.", routeName, retName) - } -} - -func TestSubRouting(t *testing.T) { - // Example from docs. - router := NewRouter() - subrouter := router.NewRoute().Host("www.domain.com").Subrouter() - route := subrouter.NewRoute().Path("/products/").Name("products") - - url := "http://www.domain.com/products/" - request, _ := http.NewRequest("GET", url, nil) - var rv RouteMatch - ok := router.Match(request, &rv) - - if !ok || rv.Route != route { - t.Errorf("Expected same route, got %+v.", rv.Route) - } - - u, _ := router.Get("products").URL() - builtUrl := u.String() - // Yay, subroute aware of the domain when building! - if builtUrl != url { - t.Errorf("Expected %q, got %q.", url, builtUrl) - } -} - -func TestVariableNames(t *testing.T) { - route := new(Route).Host("{arg1}.domain.com").Path("/{arg1}/{arg2:[0-9]+}") - if route.err == nil { - t.Errorf("Expected error for duplicated variable names") - } -} - -func TestRedirectSlash(t *testing.T) { - var route *Route - var routeMatch RouteMatch - r := NewRouter() - - r.StrictSlash(false) - route = r.NewRoute() - if route.strictSlash != false { - t.Errorf("Expected false redirectSlash.") - } - - r.StrictSlash(true) - route = r.NewRoute() - if route.strictSlash != true { - t.Errorf("Expected true redirectSlash.") - } - - route = new(Route) - route.strictSlash = true - route.Path("/{arg1}/{arg2:[0-9]+}/") - request, _ := http.NewRequest("GET", "http://localhost/foo/123", nil) - routeMatch = RouteMatch{} - _ = route.Match(request, &routeMatch) - vars := routeMatch.Vars - if vars["arg1"] != "foo" { - t.Errorf("Expected foo.") - } - if vars["arg2"] != "123" { - t.Errorf("Expected 123.") - } - rsp := NewRecorder() - routeMatch.Handler.ServeHTTP(rsp, request) - if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123/" { - t.Errorf("Expected redirect header.") - } - - route = new(Route) - route.strictSlash = true - route.Path("/{arg1}/{arg2:[0-9]+}") - request, _ = http.NewRequest("GET", "http://localhost/foo/123/", nil) - routeMatch = RouteMatch{} - _ = route.Match(request, &routeMatch) - vars = routeMatch.Vars - if vars["arg1"] != "foo" { - t.Errorf("Expected foo.") - } - if vars["arg2"] != "123" { - t.Errorf("Expected 123.") - } - rsp = NewRecorder() - routeMatch.Handler.ServeHTTP(rsp, request) - if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123" { - t.Errorf("Expected redirect header.") - } -} - -// Test for the new regexp library, still not available in stable Go. -func TestNewRegexp(t *testing.T) { - var p *routeRegexp - var matches []string - - tests := map[string]map[string][]string{ - "/{foo:a{2}}": { - "/a": nil, - "/aa": {"aa"}, - "/aaa": nil, - "/aaaa": nil, - }, - "/{foo:a{2,}}": { - "/a": nil, - "/aa": {"aa"}, - "/aaa": {"aaa"}, - "/aaaa": {"aaaa"}, - }, - "/{foo:a{2,3}}": { - "/a": nil, - "/aa": {"aa"}, - "/aaa": {"aaa"}, - "/aaaa": nil, - }, - "/{foo:[a-z]{3}}/{bar:[a-z]{2}}": { - "/a": nil, - "/ab": nil, - "/abc": nil, - "/abcd": nil, - "/abc/ab": {"abc", "ab"}, - "/abc/abc": nil, - "/abcd/ab": nil, - }, - `/{foo:\w{3,}}/{bar:\d{2,}}`: { - "/a": nil, - "/ab": nil, - "/abc": nil, - "/abc/1": nil, - "/abc/12": {"abc", "12"}, - "/abcd/12": {"abcd", "12"}, - "/abcd/123": {"abcd", "123"}, - }, - } - - for pattern, paths := range tests { - p, _ = newRouteRegexp(pattern, false, false, false, false) - for path, result := range paths { - matches = p.regexp.FindStringSubmatch(path) - if result == nil { - if matches != nil { - t.Errorf("%v should not match %v.", pattern, path) - } - } else { - if len(matches) != len(result)+1 { - t.Errorf("Expected %v matches, got %v.", len(result)+1, len(matches)) - } else { - for k, v := range result { - if matches[k+1] != v { - t.Errorf("Expected %v, got %v.", v, matches[k+1]) - } - } - } - } - } - } -} diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/regexp.go b/Godeps/_workspace/src/github.com/gorilla/mux/regexp.go deleted file mode 100644 index ef1db8ff9..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/regexp.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "bytes" - "fmt" - "net/http" - "net/url" - "regexp" - "strings" -) - -// newRouteRegexp parses a route template and returns a routeRegexp, -// used to match a host, a path or a query string. -// -// It will extract named variables, assemble a regexp to be matched, create -// a "reverse" template to build URLs and compile regexps to validate variable -// values used in URL building. -// -// Previously we accepted only Python-like identifiers for variable -// names ([a-zA-Z_][a-zA-Z0-9_]*), but currently the only restriction is that -// name and pattern can't be empty, and names can't contain a colon. -func newRouteRegexp(tpl string, matchHost, matchPrefix, matchQuery, strictSlash bool) (*routeRegexp, error) { - // Check if it is well-formed. - idxs, errBraces := braceIndices(tpl) - if errBraces != nil { - return nil, errBraces - } - // Backup the original. - template := tpl - // Now let's parse it. - defaultPattern := "[^/]+" - if matchQuery { - defaultPattern = "[^?&]+" - matchPrefix, strictSlash = true, false - } else if matchHost { - defaultPattern = "[^.]+" - matchPrefix, strictSlash = false, false - } - if matchPrefix { - strictSlash = false - } - // Set a flag for strictSlash. - endSlash := false - if strictSlash && strings.HasSuffix(tpl, "/") { - tpl = tpl[:len(tpl)-1] - endSlash = true - } - varsN := make([]string, len(idxs)/2) - varsR := make([]*regexp.Regexp, len(idxs)/2) - pattern := bytes.NewBufferString("") - if !matchQuery { - pattern.WriteByte('^') - } - reverse := bytes.NewBufferString("") - var end int - var err error - for i := 0; i < len(idxs); i += 2 { - // Set all values we are interested in. - raw := tpl[end:idxs[i]] - end = idxs[i+1] - parts := strings.SplitN(tpl[idxs[i]+1:end-1], ":", 2) - name := parts[0] - patt := defaultPattern - if len(parts) == 2 { - patt = parts[1] - } - // Name or pattern can't be empty. - if name == "" || patt == "" { - return nil, fmt.Errorf("mux: missing name or pattern in %q", - tpl[idxs[i]:end]) - } - // Build the regexp pattern. - fmt.Fprintf(pattern, "%s(%s)", regexp.QuoteMeta(raw), patt) - // Build the reverse template. - fmt.Fprintf(reverse, "%s%%s", raw) - // Append variable name and compiled pattern. - varsN[i/2] = name - varsR[i/2], err = regexp.Compile(fmt.Sprintf("^%s$", patt)) - if err != nil { - return nil, err - } - } - // Add the remaining. - raw := tpl[end:] - pattern.WriteString(regexp.QuoteMeta(raw)) - if strictSlash { - pattern.WriteString("[/]?") - } - if !matchPrefix { - pattern.WriteByte('$') - } - reverse.WriteString(raw) - if endSlash { - reverse.WriteByte('/') - } - // Compile full regexp. - reg, errCompile := regexp.Compile(pattern.String()) - if errCompile != nil { - return nil, errCompile - } - // Done! - return &routeRegexp{ - template: template, - matchHost: matchHost, - matchQuery: matchQuery, - strictSlash: strictSlash, - regexp: reg, - reverse: reverse.String(), - varsN: varsN, - varsR: varsR, - }, nil -} - -// routeRegexp stores a regexp to match a host or path and information to -// collect and validate route variables. -type routeRegexp struct { - // The unmodified template. - template string - // True for host match, false for path or query string match. - matchHost bool - // True for query string match, false for path and host match. - matchQuery bool - // The strictSlash value defined on the route, but disabled if PathPrefix was used. - strictSlash bool - // Expanded regexp. - regexp *regexp.Regexp - // Reverse template. - reverse string - // Variable names. - varsN []string - // Variable regexps (validators). - varsR []*regexp.Regexp -} - -// Match matches the regexp against the URL host or path. -func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool { - if !r.matchHost { - if r.matchQuery { - return r.regexp.MatchString(req.URL.RawQuery) - } else { - return r.regexp.MatchString(req.URL.Path) - } - } - return r.regexp.MatchString(getHost(req)) -} - -// url builds a URL part using the given values. -func (r *routeRegexp) url(pairs ...string) (string, error) { - values, err := mapFromPairs(pairs...) - if err != nil { - return "", err - } - urlValues := make([]interface{}, len(r.varsN)) - for k, v := range r.varsN { - value, ok := values[v] - if !ok { - return "", fmt.Errorf("mux: missing route variable %q", v) - } - urlValues[k] = value - } - rv := fmt.Sprintf(r.reverse, urlValues...) - if !r.regexp.MatchString(rv) { - // The URL is checked against the full regexp, instead of checking - // individual variables. This is faster but to provide a good error - // message, we check individual regexps if the URL doesn't match. - for k, v := range r.varsN { - if !r.varsR[k].MatchString(values[v]) { - return "", fmt.Errorf( - "mux: variable %q doesn't match, expected %q", values[v], - r.varsR[k].String()) - } - } - } - return rv, nil -} - -// braceIndices returns the first level curly brace indices from a string. -// It returns an error in case of unbalanced braces. -func braceIndices(s string) ([]int, error) { - var level, idx int - idxs := make([]int, 0) - for i := 0; i < len(s); i++ { - switch s[i] { - case '{': - if level++; level == 1 { - idx = i - } - case '}': - if level--; level == 0 { - idxs = append(idxs, idx, i+1) - } else if level < 0 { - return nil, fmt.Errorf("mux: unbalanced braces in %q", s) - } - } - } - if level != 0 { - return nil, fmt.Errorf("mux: unbalanced braces in %q", s) - } - return idxs, nil -} - -// ---------------------------------------------------------------------------- -// routeRegexpGroup -// ---------------------------------------------------------------------------- - -// routeRegexpGroup groups the route matchers that carry variables. -type routeRegexpGroup struct { - host *routeRegexp - path *routeRegexp - queries []*routeRegexp -} - -// setMatch extracts the variables from the URL once a route matches. -func (v *routeRegexpGroup) setMatch(req *http.Request, m *RouteMatch, r *Route) { - // Store host variables. - if v.host != nil { - hostVars := v.host.regexp.FindStringSubmatch(getHost(req)) - if hostVars != nil { - for k, v := range v.host.varsN { - m.Vars[v] = hostVars[k+1] - } - } - } - // Store path variables. - if v.path != nil { - pathVars := v.path.regexp.FindStringSubmatch(req.URL.Path) - if pathVars != nil { - for k, v := range v.path.varsN { - m.Vars[v] = pathVars[k+1] - } - // Check if we should redirect. - if v.path.strictSlash { - p1 := strings.HasSuffix(req.URL.Path, "/") - p2 := strings.HasSuffix(v.path.template, "/") - if p1 != p2 { - u, _ := url.Parse(req.URL.String()) - if p1 { - u.Path = u.Path[:len(u.Path)-1] - } else { - u.Path += "/" - } - m.Handler = http.RedirectHandler(u.String(), 301) - } - } - } - } - // Store query string variables. - rawQuery := req.URL.RawQuery - for _, q := range v.queries { - queryVars := q.regexp.FindStringSubmatch(rawQuery) - if queryVars != nil { - for k, v := range q.varsN { - m.Vars[v] = queryVars[k+1] - } - } - } -} - -// getHost tries its best to return the request host. -func getHost(r *http.Request) string { - if !r.URL.IsAbs() { - host := r.Host - // Slice off any port information. - if i := strings.Index(host, ":"); i != -1 { - host = host[:i] - } - return host - } - return r.URL.Host -} diff --git a/Godeps/_workspace/src/github.com/gorilla/mux/route.go b/Godeps/_workspace/src/github.com/gorilla/mux/route.go deleted file mode 100644 index c310e66bc..000000000 --- a/Godeps/_workspace/src/github.com/gorilla/mux/route.go +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mux - -import ( - "errors" - "fmt" - "net/http" - "net/url" - "strings" -) - -// Route stores information to match a request and build URLs. -type Route struct { - // Parent where the route was registered (a Router). - parent parentRoute - // Request handler for the route. - handler http.Handler - // List of matchers. - matchers []matcher - // Manager for the variables from host and path. - regexp *routeRegexpGroup - // If true, when the path pattern is "/path/", accessing "/path" will - // redirect to the former and vice versa. - strictSlash bool - // If true, this route never matches: it is only used to build URLs. - buildOnly bool - // The name used to build URLs. - name string - // Error resulted from building a route. - err error -} - -// Match matches the route against the request. -func (r *Route) Match(req *http.Request, match *RouteMatch) bool { - if r.buildOnly || r.err != nil { - return false - } - // Match everything. - for _, m := range r.matchers { - if matched := m.Match(req, match); !matched { - return false - } - } - // Yay, we have a match. Let's collect some info about it. - if match.Route == nil { - match.Route = r - } - if match.Handler == nil { - match.Handler = r.handler - } - if match.Vars == nil { - match.Vars = make(map[string]string) - } - // Set variables. - if r.regexp != nil { - r.regexp.setMatch(req, match, r) - } - return true -} - -// ---------------------------------------------------------------------------- -// Route attributes -// ---------------------------------------------------------------------------- - -// GetError returns an error resulted from building the route, if any. -func (r *Route) GetError() error { - return r.err -} - -// BuildOnly sets the route to never match: it is only used to build URLs. -func (r *Route) BuildOnly() *Route { - r.buildOnly = true - return r -} - -// Handler -------------------------------------------------------------------- - -// Handler sets a handler for the route. -func (r *Route) Handler(handler http.Handler) *Route { - if r.err == nil { - r.handler = handler - } - return r -} - -// HandlerFunc sets a handler function for the route. -func (r *Route) HandlerFunc(f func(http.ResponseWriter, *http.Request)) *Route { - return r.Handler(http.HandlerFunc(f)) -} - -// GetHandler returns the handler for the route, if any. -func (r *Route) GetHandler() http.Handler { - return r.handler -} - -// Name ----------------------------------------------------------------------- - -// Name sets the name for the route, used to build URLs. -// If the name was registered already it will be overwritten. -func (r *Route) Name(name string) *Route { - if r.name != "" { - r.err = fmt.Errorf("mux: route already has name %q, can't set %q", - r.name, name) - } - if r.err == nil { - r.name = name - r.getNamedRoutes()[name] = r - } - return r -} - -// GetName returns the name for the route, if any. -func (r *Route) GetName() string { - return r.name -} - -// ---------------------------------------------------------------------------- -// Matchers -// ---------------------------------------------------------------------------- - -// matcher types try to match a request. -type matcher interface { - Match(*http.Request, *RouteMatch) bool -} - -// addMatcher adds a matcher to the route. -func (r *Route) addMatcher(m matcher) *Route { - if r.err == nil { - r.matchers = append(r.matchers, m) - } - return r -} - -// addRegexpMatcher adds a host or path matcher and builder to a route. -func (r *Route) addRegexpMatcher(tpl string, matchHost, matchPrefix, matchQuery bool) error { - if r.err != nil { - return r.err - } - r.regexp = r.getRegexpGroup() - if !matchHost && !matchQuery { - if len(tpl) == 0 || tpl[0] != '/' { - return fmt.Errorf("mux: path must start with a slash, got %q", tpl) - } - if r.regexp.path != nil { - tpl = strings.TrimRight(r.regexp.path.template, "/") + tpl - } - } - rr, err := newRouteRegexp(tpl, matchHost, matchPrefix, matchQuery, r.strictSlash) - if err != nil { - return err - } - for _, q := range r.regexp.queries { - if err = uniqueVars(rr.varsN, q.varsN); err != nil { - return err - } - } - if matchHost { - if r.regexp.path != nil { - if err = uniqueVars(rr.varsN, r.regexp.path.varsN); err != nil { - return err - } - } - r.regexp.host = rr - } else { - if r.regexp.host != nil { - if err = uniqueVars(rr.varsN, r.regexp.host.varsN); err != nil { - return err - } - } - if matchQuery { - r.regexp.queries = append(r.regexp.queries, rr) - } else { - r.regexp.path = rr - } - } - r.addMatcher(rr) - return nil -} - -// Headers -------------------------------------------------------------------- - -// headerMatcher matches the request against header values. -type headerMatcher map[string]string - -func (m headerMatcher) Match(r *http.Request, match *RouteMatch) bool { - return matchMap(m, r.Header, true) -} - -// Headers adds a matcher for request header values. -// It accepts a sequence of key/value pairs to be matched. For example: -// -// r := mux.NewRouter() -// r.Headers("Content-Type", "application/json", -// "X-Requested-With", "XMLHttpRequest") -// -// The above route will only match if both request header values match. -// -// It the value is an empty string, it will match any value if the key is set. -func (r *Route) Headers(pairs ...string) *Route { - if r.err == nil { - var headers map[string]string - headers, r.err = mapFromPairs(pairs...) - return r.addMatcher(headerMatcher(headers)) - } - return r -} - -// Host ----------------------------------------------------------------------- - -// Host adds a matcher for the URL host. -// It accepts a template with zero or more URL variables enclosed by {}. -// Variables can define an optional regexp pattern to me matched: -// -// - {name} matches anything until the next dot. -// -// - {name:pattern} matches the given regexp pattern. -// -// For example: -// -// r := mux.NewRouter() -// r.Host("www.domain.com") -// r.Host("{subdomain}.domain.com") -// r.Host("{subdomain:[a-z]+}.domain.com") -// -// Variable names must be unique in a given route. They can be retrieved -// calling mux.Vars(request). -func (r *Route) Host(tpl string) *Route { - r.err = r.addRegexpMatcher(tpl, true, false, false) - return r -} - -// MatcherFunc ---------------------------------------------------------------- - -// MatcherFunc is the function signature used by custom matchers. -type MatcherFunc func(*http.Request, *RouteMatch) bool - -func (m MatcherFunc) Match(r *http.Request, match *RouteMatch) bool { - return m(r, match) -} - -// MatcherFunc adds a custom function to be used as request matcher. -func (r *Route) MatcherFunc(f MatcherFunc) *Route { - return r.addMatcher(f) -} - -// Methods -------------------------------------------------------------------- - -// methodMatcher matches the request against HTTP methods. -type methodMatcher []string - -func (m methodMatcher) Match(r *http.Request, match *RouteMatch) bool { - return matchInArray(m, r.Method) -} - -// Methods adds a matcher for HTTP methods. -// It accepts a sequence of one or more methods to be matched, e.g.: -// "GET", "POST", "PUT". -func (r *Route) Methods(methods ...string) *Route { - for k, v := range methods { - methods[k] = strings.ToUpper(v) - } - return r.addMatcher(methodMatcher(methods)) -} - -// Path ----------------------------------------------------------------------- - -// Path adds a matcher for the URL path. -// It accepts a template with zero or more URL variables enclosed by {}. The -// template must start with a "/". -// Variables can define an optional regexp pattern to me matched: -// -// - {name} matches anything until the next slash. -// -// - {name:pattern} matches the given regexp pattern. -// -// For example: -// -// r := mux.NewRouter() -// r.Path("/products/").Handler(ProductsHandler) -// r.Path("/products/{key}").Handler(ProductsHandler) -// r.Path("/articles/{category}/{id:[0-9]+}"). -// Handler(ArticleHandler) -// -// Variable names must be unique in a given route. They can be retrieved -// calling mux.Vars(request). -func (r *Route) Path(tpl string) *Route { - r.err = r.addRegexpMatcher(tpl, false, false, false) - return r -} - -// PathPrefix ----------------------------------------------------------------- - -// PathPrefix adds a matcher for the URL path prefix. This matches if the given -// template is a prefix of the full URL path. See Route.Path() for details on -// the tpl argument. -// -// Note that it does not treat slashes specially ("/foobar/" will be matched by -// the prefix "/foo") so you may want to use a trailing slash here. -// -// Also note that the setting of Router.StrictSlash() has no effect on routes -// with a PathPrefix matcher. -func (r *Route) PathPrefix(tpl string) *Route { - r.err = r.addRegexpMatcher(tpl, false, true, false) - return r -} - -// Query ---------------------------------------------------------------------- - -// Queries adds a matcher for URL query values. -// It accepts a sequence of key/value pairs. Values may define variables. -// For example: -// -// r := mux.NewRouter() -// r.Queries("foo", "bar", "id", "{id:[0-9]+}") -// -// The above route will only match if the URL contains the defined queries -// values, e.g.: ?foo=bar&id=42. -// -// It the value is an empty string, it will match any value if the key is set. -// -// Variables can define an optional regexp pattern to me matched: -// -// - {name} matches anything until the next slash. -// -// - {name:pattern} matches the given regexp pattern. -func (r *Route) Queries(pairs ...string) *Route { - length := len(pairs) - if length%2 != 0 { - r.err = fmt.Errorf( - "mux: number of parameters must be multiple of 2, got %v", pairs) - return nil - } - for i := 0; i < length; i += 2 { - if r.err = r.addRegexpMatcher(pairs[i]+"="+pairs[i+1], false, true, true); r.err != nil { - return r - } - } - - return r -} - -// Schemes -------------------------------------------------------------------- - -// schemeMatcher matches the request against URL schemes. -type schemeMatcher []string - -func (m schemeMatcher) Match(r *http.Request, match *RouteMatch) bool { - return matchInArray(m, r.URL.Scheme) -} - -// Schemes adds a matcher for URL schemes. -// It accepts a sequence of schemes to be matched, e.g.: "http", "https". -func (r *Route) Schemes(schemes ...string) *Route { - for k, v := range schemes { - schemes[k] = strings.ToLower(v) - } - return r.addMatcher(schemeMatcher(schemes)) -} - -// Subrouter ------------------------------------------------------------------ - -// Subrouter creates a subrouter for the route. -// -// It will test the inner routes only if the parent route matched. For example: -// -// r := mux.NewRouter() -// s := r.Host("www.domain.com").Subrouter() -// s.HandleFunc("/products/", ProductsHandler) -// s.HandleFunc("/products/{key}", ProductHandler) -// s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) -// -// Here, the routes registered in the subrouter won't be tested if the host -// doesn't match. -func (r *Route) Subrouter() *Router { - router := &Router{parent: r, strictSlash: r.strictSlash} - r.addMatcher(router) - return router -} - -// ---------------------------------------------------------------------------- -// URL building -// ---------------------------------------------------------------------------- - -// URL builds a URL for the route. -// -// It accepts a sequence of key/value pairs for the route variables. For -// example, given this route: -// -// r := mux.NewRouter() -// r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). -// Name("article") -// -// ...a URL for it can be built using: -// -// url, err := r.Get("article").URL("category", "technology", "id", "42") -// -// ...which will return an url.URL with the following path: -// -// "/articles/technology/42" -// -// This also works for host variables: -// -// r := mux.NewRouter() -// r.Host("{subdomain}.domain.com"). -// HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). -// Name("article") -// -// // url.String() will be "http://news.domain.com/articles/technology/42" -// url, err := r.Get("article").URL("subdomain", "news", -// "category", "technology", -// "id", "42") -// -// All variables defined in the route are required, and their values must -// conform to the corresponding patterns. -func (r *Route) URL(pairs ...string) (*url.URL, error) { - if r.err != nil { - return nil, r.err - } - if r.regexp == nil { - return nil, errors.New("mux: route doesn't have a host or path") - } - var scheme, host, path string - var err error - if r.regexp.host != nil { - // Set a default scheme. - scheme = "http" - if host, err = r.regexp.host.url(pairs...); err != nil { - return nil, err - } - } - if r.regexp.path != nil { - if path, err = r.regexp.path.url(pairs...); err != nil { - return nil, err - } - } - return &url.URL{ - Scheme: scheme, - Host: host, - Path: path, - }, nil -} - -// URLHost builds the host part of the URL for a route. See Route.URL(). -// -// The route must have a host defined. -func (r *Route) URLHost(pairs ...string) (*url.URL, error) { - if r.err != nil { - return nil, r.err - } - if r.regexp == nil || r.regexp.host == nil { - return nil, errors.New("mux: route doesn't have a host") - } - host, err := r.regexp.host.url(pairs...) - if err != nil { - return nil, err - } - return &url.URL{ - Scheme: "http", - Host: host, - }, nil -} - -// URLPath builds the path part of the URL for a route. See Route.URL(). -// -// The route must have a path defined. -func (r *Route) URLPath(pairs ...string) (*url.URL, error) { - if r.err != nil { - return nil, r.err - } - if r.regexp == nil || r.regexp.path == nil { - return nil, errors.New("mux: route doesn't have a path") - } - path, err := r.regexp.path.url(pairs...) - if err != nil { - return nil, err - } - return &url.URL{ - Path: path, - }, nil -} - -// ---------------------------------------------------------------------------- -// parentRoute -// ---------------------------------------------------------------------------- - -// parentRoute allows routes to know about parent host and path definitions. -type parentRoute interface { - getNamedRoutes() map[string]*Route - getRegexpGroup() *routeRegexpGroup -} - -// getNamedRoutes returns the map where named routes are registered. -func (r *Route) getNamedRoutes() map[string]*Route { - if r.parent == nil { - // During tests router is not always set. - r.parent = NewRouter() - } - return r.parent.getNamedRoutes() -} - -// getRegexpGroup returns regexp definitions from this route. -func (r *Route) getRegexpGroup() *routeRegexpGroup { - if r.regexp == nil { - if r.parent == nil { - // During tests router is not always set. - r.parent = NewRouter() - } - regexp := r.parent.getRegexpGroup() - if regexp == nil { - r.regexp = new(routeRegexpGroup) - } else { - // Copy. - r.regexp = &routeRegexpGroup{ - host: regexp.host, - path: regexp.path, - queries: regexp.queries, - } - } - } - return r.regexp -} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-base58/base58_test.go b/Godeps/_workspace/src/github.com/jbenet/go-base58/base58_test.go index 516781bb2..3e945b29d 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-base58/base58_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-base58/base58_test.go @@ -94,3 +94,37 @@ func TestBase58(t *testing.T) { } } } + +func BenchmarkDecodeShort(b *testing.B) { + const in = "1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = Decode(in) + } +} + +func BenchmarkEncodeShort(b *testing.B) { + var in = []byte("00eb15231dfceb60925886b67d065299925915aeb172c06647") + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = Encode(in) + } +} + +func BenchmarkDecodeOneKilo(b *testing.B) { + const in = "3GimCffBLAHhXMCeNxX2nST6dBem9pbUi3KVKykW73LmewcFtMk9oh9eNPdNR2eSzNqp7Z3E21vrWUkGHzJ7w2yqDUDJ4LKo1w5D6aafZ4SUoNQyrSVxyVG3pwgoZkKXMZVixRyiPZVUpekrsTvZuUoW7mB6BQgDTXbDuMMSRoNR7yiUTKpgwTD61DLmhNZopNxfFjn4avpYPgzsTB94iWueq1yU3EoruWCUMvp6fc1CEbDrZY3pkx9oUbUaSMC37rruBKSSGHh1ZE3XK3kQXBCFraMmUQf8dagofMEg5aTnDiLAZjLyWJMdnQwW1FqKKztP8KAQS2JX8GCCfc68KB4VGf2CfEGXtaapnsNWFrHuWi7Wo5vqyuHd21zGm1u5rsiR6tKNCsFC4nzf3WUNxJNoZrDSdF9KERqhTWWmmcM4qdKRCtBWKTrs1DJD2oiK6BK9BgwoW2dfQdKuxojFyFvmxqPKDDAEZPPpJ51wHoFzBFMM1tUBBkN15cT2GpNwKzDcjHPKJAQ6FNRgppfQytzqpq76sSeZaWAB8hhULMJCQGU57ZUjvP7xYAQwtACBnYrjdxA91XwXFbq5AsQJwAmLw6euKVWNyv11BuHrejVmnNViWg5kuZBrtgL6NtzRWHtdxngHDMtuyky3brqGXaGQhUyXrkSpeknkkHL6NLThHH5NPnfFMVPwn2xf5UM5R51X2nTBzADSVcpi4cT7i44dT7o3yRKWtKfUzZiuNyTcSSrfH8KVdLap5ZKLmdPuXM65M2Z5wJVh3Uc4iv6iZKk44RKikM7zs1hqC4sBxRwLZjxhKvvMXDjDcYFkzyUkues4y7fjdCnVTxc4vTYUqcbY2k2WMssyj9SDseVc7dVrEvWCLQtYy79mJFoz1Hmsfk5ynE28ipznzQ3yTBugLHA6j6qW3S74eY4pJ6iynFEuXT4RqqkLGFcMh3goqS7CkphUMzP4wuJyGnzqa5tCno4U3dJ2jUL7Povg8voRqYAfiHyXC8zuhn225EdmRcSnu2pAuutQVV9hN3bkjfzAFUhUWKki8SwXtFSjy6NJyrYUiaze4p7ApsjHQBCgg2zAoBaGCwVN8991Jny31B5vPyYHy1oRSE4xTVZ7tTw9FyQ7w9p1NSEF4sziCxZHh5rFWZKAajc5c7KaMNDvHPNV6S62MTFGTyuKPQNbv9yHRGN4eH6SnZGW6snvEVdYCspWZ1U3Nbxo6vCmBK95UyYpcxHgg1CCGdU4s3edju2NDQkMifyPkJdkabzzHVDhJJbChAJc1ACQfNW74VXXwrBZmeZyA2R28MBctDyXuSuffiwueys2LVowLu9wiTHUox7KQjtHK2c9howk9czzx2mpnYzkVYH42CYsWa5514EM4CJEXPJSSbXSgJJ" + b.SetBytes(int64(len(in))) // 1024 + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = Decode(in) + } +} + +func BenchmarkEncodeOneKilo(b *testing.B) { + var in = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x00\x00\x04\xff\xfb\x63\xc9\x7e\x5f\x97\x68\xe5\x10\x08\xe5\xa5\x9a\x7c\x24\x75\x35\xbe\xaf\x37\x0b\xc3\xf4\x18\x62\x4a\xb7\x18\xd0\x10\x0b\x6b\x62\xe1\x36\x0f\x62\xa6\xeb\x78\xa5\xf3\x33\x52\xbc\xdf\x04\xcb\x37\xcf\x3d\x97\x5c\xb8\x75\x09\x1f\x18\x9f\xfc\xa9\xda\x1e\x59\x77\x09\x9c\x5d\xb6\xf2\x9e\x45\xb7\x5e\x5d\x11\xf1\x20\x14\x85\xf8\x54\x87\x8c\x1e\x2c\x2e\x15\x57\x89\xe7\x5d\x49\xb6\xae\x24\x3a\x20\x50\x0e\xa7\x5b\x10\xbf\x0a\xb4\x01\x42\xed\xce\x2d\x45\x21\xb6\xe8\x64\x73\x4e\x7e\x0a\x36\x1d\x57\x0a\x5e\x1c\x21\xc2\xb8\xe7\x89\x82\xe4\x04\x7e\x50\xff\xda\x4f\xfe\x11\x95\xfb\x35\xf9\x6d\x32\xce\xef\x8f\x3d\x1b\xdb\x38\xfa\xcd\x26\x36\x12\x93\xa0\x96\xea\x42\xbe\xd6\x85\x86\xc1\xc1\xe2\x55\x41\xd1\x7f\x8d\x0e\x00\x81\x58\xb4\x10\xbb\x64\x92\x05\x07\xa9\xd5\xd9\x40\x28\x8b\x9b\x4c\x8d\x8e\x4e\x69\xf9\xc9\x35\xea\xda\x2f\x61\x87\x35\x2d\x6b\x25\x32\xf0\x7e\x89\x1a\xcb\xc0\xea\x66\x88\x99\x39\xe0\x3b\x24\x3b\x05\x74\xd3\x72\xf6\x48\x15\xdc\x02\x0a\xbf\xc8\x49\x42\x10\x22\xeb\xe9\x44\x71\x55\xaf\x67\x67\xe6\x2a\x40\x31\x81\xb9\x6f\x65\x86\x0f\x0f\x9d\x58\x4c\x51\xc1\x2e\x4e\x60\x7e\xe8\x93\x39\x90\xda\xe5\xbe\xec\xe4\xdd\xbc\x1d\xba\x40\xa6\x85\xd9\xb2\xec\xb4\x26\x74\xee\xc1\xec\xe3\x40\xb9\x49\xa3\xe1\x26\x76\x8a\xeb\x95\xc8\x72\xb0\x85\x36\x19\x3f\x55\x06\x7b\xcd\x3e\xd0\xdf\x7e\x8d\x2a\xea\xa6\x24\xc6\xf6\xfb\xda\xe0\x45\xcf\x32\x0e\xbc\xf4\x41\x7d\x71\x3d\x86\xf9\xb4\xaf\x07\xa0\xd1\x34\x8a\x02\x28\x56\xd4\xcc\x36\x44\x98\x44\xcb\x9d\xc5\xfc\x45\x2d\xc4\x5c\xfe\xce\xaa\x44\xda\x66\x52\x2d\x32\x6e\x13\x32\xac\xaf\x13\x72\x87\x79\xd2\x92\x54\x9f\xc7\xb9\xf3\x21\xae\xdd\x69\x44\xe9\x46\x94\x1c\x62\x84\x03\xe0\xbf\x66\xfb\xe0\x79\xf9\x57\x9e\x22\x9e\x23\x2d\x2a\x73\xeb\x74\x38\xf0\xea\x5d\xb3\x8f\x87\x26\x3e\x3c\x54\x11\xb7\x98\xbd\x7f\x78\x64\xa3\xf1\x8f\xa9\x5e\x4f\x18\x3f\xa7\x1f\x3a\x29\x27\x27\xb7\x49\x40\x16\x18\x1f\xd3\xed\x86\x61\xbd\xc3\x4e\x4a\x53\x37\x78\x5c\x00\xd3\x50\x45\x1c\x55\xc0\x9b\xd7\x62\x29\x88\x2e\xa4\x0d\x6a\x15\x6c\x33\x3c\xe7\x31\xfa\xc1\xaf\xdf\x7a\x3e\x37\x3e\xe5\xbc\xfd\xfb\x9b\x72\x10\x35\x90\x25\x6e\x87\x0d\x74\x1c\xfd\xe3\x0b\xee\xf5\x92\x28\x8d\x22\x8a\x49\x7b\xcd\xbb\xd8\x24\x6b\x5e\x58\x40\xec\x1b\x6c\xed\x8e\xcb\x56\x62\xa6\xb4\x42\x3d\x7d\xa2\xef\x27\x27\x46\x50\xbc\x5e\x37\x9b\x27\x72\xf0\xea\xa7\xe7\x4d\xf4\xae\x7e\x95\x8f\x91\x2e\x58\xc4\x6a\x06\xda\x7a\x06\x5c\x8d\xfe\xef\xf5\xb3\x0f\xb4\x0a\x20\x53\xd8\x35\x80\x02\xca\x97\x81\xb6\x1c\x4b\x8f\xb7\xee\xd0\xc3\x88\x6c\x76\x3e\xb0\x28\xce\xa1\x9f\x76\x5f\xaa\xc3\x53\x44\x09\x70\xa3\x95\xd9\x8c\x54\xba\x8a\x9a\x6b\xce\xc3\x07\xdf\x13\x6d\xea\x0f\x51\x9c\xe2\x81\x87\xf6\x82\x7a\x70\xd8\xfa\xe2\xa8\x32\xc1\x5e\x53\xc2\x85\xe9\x61\x8a\x17\x82\x12\xab\x92\x79\x2b\xed\x07\xca\x1e\x93\x23\x9c\x4b\xd2\x89\x86\xac\x55\xf9\x50\x23\x8f\x9e\xd3\xab\x22\x57\x91\x5a\x0b\x48\xd7\xa2\xb8\x06\xbb\x74\xae\xe9\xca\x06\x41\x8d\x6a\x00\x42\xc4\x40\xa9\xfe\xae\x88\x42\xc2\x83\xe0\x8a\xd8\x5c\xbb\x5a\xb5\x9c\x1d\xa5\xbe\x67\x50\xb1\x4e\xec\x96\x65\xaa\x87\x5b\xb0\x76\x88\xe3\x1b\xcb\x38\x21\x02\x8e\xc9\xe7\xf5\xc7\xe1\x1d\xe8\xeb\x54\x0e\x0b\xea\xd1\x2e\xad\xbb\xec\x22\x21\xb3\x64\x36\x29\x34\x5e\x3a\x22\xe8\x03\x4b\x86\xb1\x67\x7d\x4f\x48\x6d\xfb\x4b\xde\xe6\x4c\xb0\xaf\x40\x66\xab\xe9\x1a\x4e\xae\x1a\x7e\x05\xc5\x67\x2a\x95\x6d\xc2\x61\x35\x20\xfe\x33\xc3\x2c\x7f\x9b\xbe\x9f\x9a\xd5\xf0\x63\x28\xa1\x94\xb1\x5c\xc1\x18\x6b\x5b\x33\xb4\x4d\xcf\xbe\xf7\xb2\x94\x58\xaa\xcf\xad\xc8\x75\x93\x1a\x08\xf4\xd2\xd9\xf6\x95\x03\x3b\xf3\x4e\xfb\x15\xe4\x28\xed\xd5\x79\xd9\xbf\xb7\x8f\xb2\x70\x16\x4c\x2d\x65\xf6\xec\x33\x1e\xaf\xea\x46\x69\xc6\x9a\x6b\xdd\xf3\x57\xe0\x1d\x28\xcd\xf8\x83\x3d\x94\x4c\x2f\x6e\xfd\x51\x3d\xa8\xff\xcb\x33\xad\x32\x42\x0e\xd3\x00\x0a\xe5\x71\x76\x3b\x83\xc9\x2a\x67\x50\xc3\xa5\xeb\x4d\x8d\x67\xd6\xd9\x1b\x9a\x5a\xbe\xdd\xc5\x15\x00\xcf\x97\x0f\x47\x44\x34\x1d\x4e\xb6\x6f\x91\x31\xf3\x45\x0f\x59\x48\x10\x23\x53\x40\x49\x83\xe6\xc8\xdf\x51\x6c\xa8\x9f\x3a\x43\x3d\xb9\xd4\xea\x30\x4d\xe0\xd2\xb8\x44\xf3\x91\x20\x79\xdb\x7b\xe6\x50\xf9\x0f\xfb\x4c\xac\x79\x93\xf6\xf8\x96\x0d\x55\x7c\x41\x9b\x1a\x86\xad\x4b\xd1\xf9\x5d\xed\x3a\x4f\xc9\x64\x72\xd4\x22\x53\x59\x2f\x01\x00\x00\xff\xff\xc6\xfd\xa0\x37\x00\x04\x00\x00") + b.SetBytes(int64(len(in))) // 1024 + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = Encode(in) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/available_unix.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/available_unix.go index 98d452988..fb52a87cd 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/available_unix.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/available_unix.go @@ -1,4 +1,5 @@ // +build darwin freebsd dragonfly netbsd openbsd linux + package reuseport import ( diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_windows.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_windows.go index d1dcfbdc4..c33b2eed7 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_windows.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/impl_windows.go @@ -1,8 +1,6 @@ package reuseport -import ( - "net" -) +import "net" // TODO. for now, just pass it over to net.Listen/net.Dial @@ -10,6 +8,14 @@ func listen(network, address string) (net.Listener, error) { return net.Listen(network, address) } +func listenPacket(netw, laddr string) (net.PacketConn, error) { + return net.ListenPacket(netw, laddr) +} + +func listenStream(netw, addr string) (net.Listener, error) { + return listen(netw, addr) +} + func dial(dialer net.Dialer, network, address string) (net.Conn, error) { return dialer.Dial(network, address) } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/opts_posix.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/opts_posix.go index de122f46d..cc5774ff3 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/opts_posix.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/opts_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows +// +build darwin dragonfly freebsd linux netbsd openbsd solaris package reuseport diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_bsd.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_bsd.go index 058f21b85..bac906f9c 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_bsd.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_bsd.go @@ -1,4 +1,4 @@ -// +build darwin freebsd dragonfly netbsd openbsd +// +build darwin,amd64 freebsd dragonfly netbsd openbsd package poll diff --git a/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_darwin_386.go b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_darwin_386.go new file mode 100644 index 000000000..6cff81dfd --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-reuseport/poll/poll_darwin_386.go @@ -0,0 +1,57 @@ +package poll + +import ( + "syscall" + "time" +) + +type Poller struct { + kqfd int + event syscall.Kevent_t +} + +func New(fd int) (p *Poller, err error) { + p = &Poller{} + + p.kqfd, err = syscall.Kqueue() + if p.kqfd == -1 || err != nil { + return nil, err + } + + p.event = syscall.Kevent_t{ + Ident: uint32(fd), + Filter: syscall.EVFILT_WRITE, + Flags: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, + Fflags: 0, + Data: 0, + Udata: nil, + } + return p, nil +} + +func (p *Poller) Close() error { + return syscall.Close(p.kqfd) +} + +func (p *Poller) WaitWrite(deadline time.Time) error { + + // setup timeout + var timeout *syscall.Timespec + if !deadline.IsZero() { + d := deadline.Sub(time.Now()) + t := syscall.NsecToTimespec(d.Nanoseconds()) + timeout = &t + } + + // wait on kevent + events := make([]syscall.Kevent_t, 1) + n, err := syscall.Kevent(p.kqfd, []syscall.Kevent_t{p.event}, events, timeout) + if err != nil { + return err + } + + if n < 1 { + return errTimeout + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go index a50021ad5..07a2a09e7 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go @@ -11,12 +11,18 @@ import ( "fmt" "os" "runtime" + "strings" ) func executable() (string, error) { switch runtime.GOOS { case "linux": - return os.Readlink("/proc/self/exe") + const deletedSuffix = " (deleted)" + execpath, err := os.Readlink("/proc/self/exe") + if err != nil { + return execpath, err + } + return strings.TrimSuffix(execpath, deletedSuffix), nil case "netbsd": return os.Readlink("/proc/curproc/exe") case "openbsd", "dragonfly": diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go index dc661dbc2..5aafa3af2 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go @@ -7,35 +7,42 @@ package osext import ( + "bytes" "fmt" + "io" "os" - oexec "os/exec" + "os/exec" "path/filepath" "runtime" "testing" ) -const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH" +const ( + executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE" -func TestExecPath(t *testing.T) { + executableEnvValueMatch = "match" + executableEnvValueDelete = "delete" +) + +func TestExecutableMatch(t *testing.T) { ep, err := Executable() if err != nil { - t.Fatalf("ExecPath failed: %v", err) + t.Fatalf("Executable failed: %v", err) } - // we want fn to be of the form "dir/prog" + + // fullpath to be of the form "dir/prog". dir := filepath.Dir(filepath.Dir(ep)) - fn, err := filepath.Rel(dir, ep) + fullpath, err := filepath.Rel(dir, ep) if err != nil { t.Fatalf("filepath.Rel: %v", err) } - cmd := &oexec.Cmd{} - // make child start with a relative program path - cmd.Dir = dir - cmd.Path = fn - // forge argv[0] for child, so that we can verify we could correctly - // get real path of the executable without influenced by argv[0]. - cmd.Args = []string{"-", "-test.run=XXXX"} - cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)} + // Make child start with a relative program path. + // Alter argv[0] for child to verify getting real path without argv[0]. + cmd := &exec.Cmd{ + Dir: dir, + Path: fullpath, + Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)}, + } out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("exec(self) failed: %v", err) @@ -49,6 +56,63 @@ func TestExecPath(t *testing.T) { } } +func TestExecutableDelete(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + fpath, err := Executable() + if err != nil { + t.Fatalf("Executable failed: %v", err) + } + + r, w := io.Pipe() + stderrBuff := &bytes.Buffer{} + stdoutBuff := &bytes.Buffer{} + cmd := &exec.Cmd{ + Path: fpath, + Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)}, + Stdin: r, + Stderr: stderrBuff, + Stdout: stdoutBuff, + } + err = cmd.Start() + if err != nil { + t.Fatalf("exec(self) start failed: %v", err) + } + + tempPath := fpath + "_copy" + _ = os.Remove(tempPath) + + err = copyFile(tempPath, fpath) + if err != nil { + t.Fatalf("copy file failed: %v", err) + } + err = os.Remove(fpath) + if err != nil { + t.Fatalf("remove running test file failed: %v", err) + } + err = os.Rename(tempPath, fpath) + if err != nil { + t.Fatalf("rename copy to previous name failed: %v", err) + } + + w.Write([]byte{0}) + w.Close() + + err = cmd.Wait() + if err != nil { + t.Fatalf("exec wait failed: %v", err) + } + + childPath := stderrBuff.String() + if !filepath.IsAbs(childPath) { + t.Fatalf("Child returned %q, want an absolute path", childPath) + } + if !sameFile(childPath, fpath) { + t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath) + } +} + func sameFile(fn1, fn2 string) bool { fi1, err := os.Stat(fn1) if err != nil { @@ -60,10 +124,30 @@ func sameFile(fn1, fn2 string) bool { } return os.SameFile(fi1, fi2) } +func copyFile(dest, src string) error { + df, err := os.Create(dest) + if err != nil { + return err + } + defer df.Close() -func init() { - if e := os.Getenv(execPath_EnvVar); e != "" { - // first chdir to another path + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + _, err = io.Copy(df, sf) + return err +} + +func TestMain(m *testing.M) { + env := os.Getenv(executableEnvVar) + switch env { + case "": + os.Exit(m.Run()) + case executableEnvValueMatch: + // First chdir to another path. dir := "/" if runtime.GOOS == "windows" { dir = filepath.VolumeName(".") @@ -74,6 +158,23 @@ func init() { } else { fmt.Fprint(os.Stderr, ep) } - os.Exit(0) + case executableEnvValueDelete: + bb := make([]byte, 1) + var err error + n, err := os.Stdin.Read(bb) + if err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + os.Exit(2) + } + if n != 1 { + fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n) + os.Exit(2) + } + if ep, err := Executable(); err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + } else { + fmt.Fprint(os.Stderr, ep) + } } + os.Exit(0) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go similarity index 98% rename from Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go rename to Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go index e76657e5e..0dd60fd82 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go @@ -4,7 +4,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// +build go1.3 +// +build !go1.2 package leveldb diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go new file mode 100644 index 000000000..175e22203 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2012, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !go1.2 + +package cache + +import ( + "math/rand" + "testing" +) + +func BenchmarkLRUCache(b *testing.B) { + c := NewCache(NewLRU(10000)) + + b.SetParallelism(10) + b.RunParallel(func(pb *testing.PB) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for pb.Next() { + key := uint64(r.Intn(1000000)) + c.Get(0, key, func() (int, Value) { + return 1, key + }).Release() + } + }) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go index baced7717..5fb4b910b 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go @@ -8,152 +8,669 @@ package cache import ( + "sync" "sync/atomic" + "unsafe" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util" ) -// SetFunc is the function that will be called by Namespace.Get to create -// a cache object, if charge is less than one than the cache object will -// not be registered to cache tree, if value is nil then the cache object -// will not be created. -type SetFunc func() (charge int, value interface{}) - -// DelFin is the function that will be called as the result of a delete operation. -// Exist == true is indication that the object is exist, and pending == true is -// indication of deletion already happen but haven't done yet (wait for all handles -// to be released). And exist == false means the object doesn't exist. -type DelFin func(exist, pending bool) - -// PurgeFin is the function that will be called as the result of a purge operation. -type PurgeFin func(ns, key uint64) - -// Cache is a cache tree. A cache instance must be goroutine-safe. -type Cache interface { - // SetCapacity sets cache tree capacity. - SetCapacity(capacity int) - - // Capacity returns cache tree capacity. +// Cacher provides interface to implements a caching functionality. +// An implementation must be goroutine-safe. +type Cacher interface { + // Capacity returns cache capacity. Capacity() int - // Used returns used cache tree capacity. - Used() int + // SetCapacity sets cache capacity. + SetCapacity(capacity int) - // Size returns entire alive cache objects size. - Size() int + // Promote promotes the 'cache node'. + Promote(n *Node) - // NumObjects returns number of alive objects. - NumObjects() int + // Ban evicts the 'cache node' and prevent subsequent 'promote'. + Ban(n *Node) - // GetNamespace gets cache namespace with the given id. - // GetNamespace is never return nil. - GetNamespace(id uint64) Namespace + // Evict evicts the 'cache node'. + Evict(n *Node) - // PurgeNamespace purges cache namespace with the given id from this cache tree. - // Also read Namespace.Purge. - PurgeNamespace(id uint64, fin PurgeFin) + // EvictNS evicts 'cache node' with the given namespace. + EvictNS(ns uint64) - // ZapNamespace detaches cache namespace with the given id from this cache tree. - // Also read Namespace.Zap. - ZapNamespace(id uint64) + // EvictAll evicts all 'cache node'. + EvictAll() - // Purge purges all cache namespace from this cache tree. - // This is behave the same as calling Namespace.Purge method on all cache namespace. - Purge(fin PurgeFin) - - // Zap detaches all cache namespace from this cache tree. - // This is behave the same as calling Namespace.Zap method on all cache namespace. - Zap() + // Close closes the 'cache tree' + Close() error } -// Namespace is a cache namespace. A namespace instance must be goroutine-safe. -type Namespace interface { - // Get gets cache object with the given key. - // If cache object is not found and setf is not nil, Get will atomically creates - // the cache object by calling setf. Otherwise Get will returns nil. - // - // The returned cache handle should be released after use by calling Release - // method. - Get(key uint64, setf SetFunc) Handle +// Value is a 'cacheable object'. It may implements util.Releaser, if +// so the the Release method will be called once object is released. +type Value interface{} - // Delete removes cache object with the given key from cache tree. - // A deleted cache object will be released as soon as all of its handles have - // been released. - // Delete only happen once, subsequent delete will consider cache object doesn't - // exist, even if the cache object ins't released yet. - // - // If not nil, fin will be called if the cache object doesn't exist or when - // finally be released. - // - // Delete returns true if such cache object exist and never been deleted. - Delete(key uint64, fin DelFin) bool - - // Purge removes all cache objects within this namespace from cache tree. - // This is the same as doing delete on all cache objects. - // - // If not nil, fin will be called on all cache objects when its finally be - // released. - Purge(fin PurgeFin) - - // Zap detaches namespace from cache tree and release all its cache objects. - // A zapped namespace can never be filled again. - // Calling Get on zapped namespace will always return nil. - Zap() +type CacheGetter struct { + Cache *Cache + NS uint64 } -// Handle is a cache handle. -type Handle interface { - // Release releases this cache handle. This method can be safely called mutiple - // times. - Release() - - // Value returns value of this cache handle. - // Value will returns nil after this cache handle have be released. - Value() interface{} +func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle { + return g.Cache.Get(g.NS, key, setFunc) } +// The hash tables implementation is based on: +// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014. + const ( - DelNotExist = iota - DelExist - DelPendig + mInitialSize = 1 << 4 + mOverflowThreshold = 1 << 5 + mOverflowGrowThreshold = 1 << 7 ) -// Namespace state. -type nsState int - -const ( - nsEffective nsState = iota - nsZapped -) - -// Node state. -type nodeState int - -const ( - nodeZero nodeState = iota - nodeEffective - nodeEvicted - nodeDeleted -) - -// Fake handle. -type fakeHandle struct { - value interface{} - fin func() - once uint32 +type mBucket struct { + mu sync.Mutex + node []*Node + frozen bool } -func (h *fakeHandle) Value() interface{} { - if atomic.LoadUint32(&h.once) == 0 { - return h.value +func (b *mBucket) freeze() []*Node { + b.mu.Lock() + defer b.mu.Unlock() + if !b.frozen { + b.frozen = true + } + return b.node +} + +func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) { + b.mu.Lock() + + if b.frozen { + b.mu.Unlock() + return + } + + // Scan the node. + for _, n := range b.node { + if n.hash == hash && n.ns == ns && n.key == key { + atomic.AddInt32(&n.ref, 1) + b.mu.Unlock() + return true, false, n + } + } + + // Get only. + if noset { + b.mu.Unlock() + return true, false, nil + } + + // Create node. + n = &Node{ + r: r, + hash: hash, + ns: ns, + key: key, + ref: 1, + } + // Add node to bucket. + b.node = append(b.node, n) + bLen := len(b.node) + b.mu.Unlock() + + // Update counter. + grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold + if bLen > mOverflowThreshold { + grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold + } + + // Grow. + if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { + nhLen := len(h.buckets) << 1 + nh := &mNode{ + buckets: make([]unsafe.Pointer, nhLen), + mask: uint32(nhLen) - 1, + pred: unsafe.Pointer(h), + growThreshold: int32(nhLen * mOverflowThreshold), + shrinkThreshold: int32(nhLen >> 1), + } + ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) + if !ok { + panic("BUG: failed swapping head") + } + go nh.initBuckets() + } + + return true, true, n +} + +func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) { + b.mu.Lock() + + if b.frozen { + b.mu.Unlock() + return + } + + // Scan the node. + var ( + n *Node + bLen int + ) + for i := range b.node { + n = b.node[i] + if n.ns == ns && n.key == key { + if atomic.LoadInt32(&n.ref) == 0 { + deleted = true + + // Call releaser. + if n.value != nil { + if r, ok := n.value.(util.Releaser); ok { + r.Release() + } + n.value = nil + } + + // Remove node from bucket. + b.node = append(b.node[:i], b.node[i+1:]...) + bLen = len(b.node) + } + break + } + } + b.mu.Unlock() + + if deleted { + // Call OnDel. + for _, f := range n.onDel { + f() + } + + // Update counter. + atomic.AddInt32(&r.size, int32(n.size)*-1) + shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold + if bLen >= mOverflowThreshold { + atomic.AddInt32(&h.overflow, -1) + } + + // Shrink. + if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { + nhLen := len(h.buckets) >> 1 + nh := &mNode{ + buckets: make([]unsafe.Pointer, nhLen), + mask: uint32(nhLen) - 1, + pred: unsafe.Pointer(h), + growThreshold: int32(nhLen * mOverflowThreshold), + shrinkThreshold: int32(nhLen >> 1), + } + ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) + if !ok { + panic("BUG: failed swapping head") + } + go nh.initBuckets() + } + } + + return true, deleted +} + +type mNode struct { + buckets []unsafe.Pointer // []*mBucket + mask uint32 + pred unsafe.Pointer // *mNode + resizeInProgess int32 + + overflow int32 + growThreshold int32 + shrinkThreshold int32 +} + +func (n *mNode) initBucket(i uint32) *mBucket { + if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil { + return b + } + + p := (*mNode)(atomic.LoadPointer(&n.pred)) + if p != nil { + var node []*Node + if n.mask > p.mask { + // Grow. + pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask])) + if pb == nil { + pb = p.initBucket(i & p.mask) + } + m := pb.freeze() + // Split nodes. + for _, x := range m { + if x.hash&n.mask == i { + node = append(node, x) + } + } + } else { + // Shrink. + pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i])) + if pb0 == nil { + pb0 = p.initBucket(i) + } + pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))])) + if pb1 == nil { + pb1 = p.initBucket(i + uint32(len(n.buckets))) + } + m0 := pb0.freeze() + m1 := pb1.freeze() + // Merge nodes. + node = make([]*Node, 0, len(m0)+len(m1)) + node = append(node, m0...) + node = append(node, m1...) + } + b := &mBucket{node: node} + if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) { + if len(node) > mOverflowThreshold { + atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold)) + } + return b + } + } + + return (*mBucket)(atomic.LoadPointer(&n.buckets[i])) +} + +func (n *mNode) initBuckets() { + for i := range n.buckets { + n.initBucket(uint32(i)) + } + atomic.StorePointer(&n.pred, nil) +} + +// Cache is a 'cache map'. +type Cache struct { + mu sync.RWMutex + mHead unsafe.Pointer // *mNode + nodes int32 + size int32 + cacher Cacher + closed bool +} + +// NewCache creates a new 'cache map'. The cacher is optional and +// may be nil. +func NewCache(cacher Cacher) *Cache { + h := &mNode{ + buckets: make([]unsafe.Pointer, mInitialSize), + mask: mInitialSize - 1, + growThreshold: int32(mInitialSize * mOverflowThreshold), + shrinkThreshold: 0, + } + for i := range h.buckets { + h.buckets[i] = unsafe.Pointer(&mBucket{}) + } + r := &Cache{ + mHead: unsafe.Pointer(h), + cacher: cacher, + } + return r +} + +func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) { + h := (*mNode)(atomic.LoadPointer(&r.mHead)) + i := hash & h.mask + b := (*mBucket)(atomic.LoadPointer(&h.buckets[i])) + if b == nil { + b = h.initBucket(i) + } + return h, b +} + +func (r *Cache) delete(n *Node) bool { + for { + h, b := r.getBucket(n.hash) + done, deleted := b.delete(r, h, n.hash, n.ns, n.key) + if done { + return deleted + } + } + return false +} + +// Nodes returns number of 'cache node' in the map. +func (r *Cache) Nodes() int { + return int(atomic.LoadInt32(&r.nodes)) +} + +// Size returns sums of 'cache node' size in the map. +func (r *Cache) Size() int { + return int(atomic.LoadInt32(&r.size)) +} + +// Capacity returns cache capacity. +func (r *Cache) Capacity() int { + if r.cacher == nil { + return 0 + } + return r.cacher.Capacity() +} + +// SetCapacity sets cache capacity. +func (r *Cache) SetCapacity(capacity int) { + if r.cacher != nil { + r.cacher.SetCapacity(capacity) + } +} + +// Get gets 'cache node' with the given namespace and key. +// If cache node is not found and setFunc is not nil, Get will atomically creates +// the 'cache node' by calling setFunc. Otherwise Get will returns nil. +// +// The returned 'cache handle' should be released after use by calling Release +// method. +func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return nil + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, setFunc == nil) + if done { + if n != nil { + n.mu.Lock() + if n.value == nil { + if setFunc == nil { + n.mu.Unlock() + n.unref() + return nil + } + + n.size, n.value = setFunc() + if n.value == nil { + n.size = 0 + n.mu.Unlock() + n.unref() + return nil + } + atomic.AddInt32(&r.size, int32(n.size)) + } + n.mu.Unlock() + if r.cacher != nil { + r.cacher.Promote(n) + } + return &Handle{unsafe.Pointer(n)} + } + + break + } } return nil } -func (h *fakeHandle) Release() { - if !atomic.CompareAndSwapUint32(&h.once, 0, 1) { +// Delete removes and ban 'cache node' with the given namespace and key. +// A banned 'cache node' will never inserted into the 'cache tree'. Ban +// only attributed to the particular 'cache node', so when a 'cache node' +// is recreated it will not be banned. +// +// If onDel is not nil, then it will be executed if such 'cache node' +// doesn't exist or once the 'cache node' is released. +// +// Delete return true is such 'cache node' exist. +func (r *Cache) Delete(ns, key uint64, onDel func()) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return false + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, true) + if done { + if n != nil { + if onDel != nil { + n.mu.Lock() + n.onDel = append(n.onDel, onDel) + n.mu.Unlock() + } + if r.cacher != nil { + r.cacher.Ban(n) + } + n.unref() + return true + } + + break + } + } + + if onDel != nil { + onDel() + } + + return false +} + +// Evict evicts 'cache node' with the given namespace and key. This will +// simply call Cacher.Evict. +// +// Evict return true is such 'cache node' exist. +func (r *Cache) Evict(ns, key uint64) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return false + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, true) + if done { + if n != nil { + if r.cacher != nil { + r.cacher.Evict(n) + } + n.unref() + return true + } + + break + } + } + + return false +} + +// EvictNS evicts 'cache node' with the given namespace. This will +// simply call Cacher.EvictNS. +func (r *Cache) EvictNS(ns uint64) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { return } - if h.fin != nil { - h.fin() - h.fin = nil + + if r.cacher != nil { + r.cacher.EvictNS(ns) } } + +// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll. +func (r *Cache) EvictAll() { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return + } + + if r.cacher != nil { + r.cacher.EvictAll() + } +} + +// Close closes the 'cache map' and releases all 'cache node'. +func (r *Cache) Close() error { + r.mu.Lock() + if !r.closed { + r.closed = true + + if r.cacher != nil { + if err := r.cacher.Close(); err != nil { + return err + } + } + + h := (*mNode)(r.mHead) + h.initBuckets() + + for i := range h.buckets { + b := (*mBucket)(h.buckets[i]) + for _, n := range b.node { + // Call releaser. + if n.value != nil { + if r, ok := n.value.(util.Releaser); ok { + r.Release() + } + n.value = nil + } + + // Call OnDel. + for _, f := range n.onDel { + f() + } + } + } + } + r.mu.Unlock() + return nil +} + +// Node is a 'cache node'. +type Node struct { + r *Cache + + hash uint32 + ns, key uint64 + + mu sync.Mutex + size int + value Value + + ref int32 + onDel []func() + + CacheData unsafe.Pointer +} + +// NS returns this 'cache node' namespace. +func (n *Node) NS() uint64 { + return n.ns +} + +// Key returns this 'cache node' key. +func (n *Node) Key() uint64 { + return n.key +} + +// Size returns this 'cache node' size. +func (n *Node) Size() int { + return n.size +} + +// Value returns this 'cache node' value. +func (n *Node) Value() Value { + return n.value +} + +// Ref returns this 'cache node' ref counter. +func (n *Node) Ref() int32 { + return atomic.LoadInt32(&n.ref) +} + +// GetHandle returns an handle for this 'cache node'. +func (n *Node) GetHandle() *Handle { + if atomic.AddInt32(&n.ref, 1) <= 1 { + panic("BUG: Node.GetHandle on zero ref") + } + return &Handle{unsafe.Pointer(n)} +} + +func (n *Node) unref() { + if atomic.AddInt32(&n.ref, -1) == 0 { + n.r.delete(n) + } +} + +func (n *Node) unrefLocked() { + if atomic.AddInt32(&n.ref, -1) == 0 { + n.r.mu.RLock() + if !n.r.closed { + n.r.delete(n) + } + n.r.mu.RUnlock() + } +} + +type Handle struct { + n unsafe.Pointer // *Node +} + +func (h *Handle) Value() Value { + n := (*Node)(atomic.LoadPointer(&h.n)) + if n != nil { + return n.value + } + return nil +} + +func (h *Handle) Release() { + nPtr := atomic.LoadPointer(&h.n) + if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) { + n := (*Node)(nPtr) + n.unrefLocked() + } +} + +func murmur32(ns, key uint64, seed uint32) uint32 { + const ( + m = uint32(0x5bd1e995) + r = 24 + ) + + k1 := uint32(ns >> 32) + k2 := uint32(ns) + k3 := uint32(key >> 32) + k4 := uint32(key) + + k1 *= m + k1 ^= k1 >> r + k1 *= m + + k2 *= m + k2 ^= k2 >> r + k2 *= m + + k3 *= m + k3 ^= k3 >> r + k3 *= m + + k4 *= m + k4 ^= k4 >> r + k4 *= m + + h := seed + + h *= m + h ^= k1 + h *= m + h ^= k2 + h *= m + h ^= k3 + h *= m + h ^= k4 + + h ^= h >> 13 + h *= m + h ^= h >> 15 + + return h +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go index 865bc5733..c2a50156f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go @@ -13,11 +13,26 @@ import ( "sync/atomic" "testing" "time" + "unsafe" ) +type int32o int32 + +func (o *int32o) acquire() { + if atomic.AddInt32((*int32)(o), 1) != 1 { + panic("BUG: invalid ref") + } +} + +func (o *int32o) Release() { + if atomic.AddInt32((*int32)(o), -1) != 0 { + panic("BUG: invalid ref") + } +} + type releaserFunc struct { fn func() - value interface{} + value Value } func (r releaserFunc) Release() { @@ -26,8 +41,8 @@ func (r releaserFunc) Release() { } } -func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle { - return ns.Get(key, func() (int, interface{}) { +func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle { + return c.Get(ns, key, func() (int, Value) { if relf != nil { return charge, releaserFunc{relf, value} } else { @@ -36,7 +51,246 @@ func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) H }) } -func TestCache_HitMiss(t *testing.T) { +func TestCacheMap(t *testing.T) { + runtime.GOMAXPROCS(runtime.NumCPU()) + + nsx := []struct { + nobjects, nhandles, concurrent, repeat int + }{ + {10000, 400, 50, 3}, + {100000, 1000, 100, 10}, + } + + var ( + objects [][]int32o + handles [][]unsafe.Pointer + ) + + for _, x := range nsx { + objects = append(objects, make([]int32o, x.nobjects)) + handles = append(handles, make([]unsafe.Pointer, x.nhandles)) + } + + c := NewCache(nil) + + wg := new(sync.WaitGroup) + var done int32 + + for ns, x := range nsx { + for i := 0; i < x.concurrent; i++ { + wg.Add(1) + go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for j := len(objects) * repeat; j >= 0; j-- { + key := uint64(r.Intn(len(objects))) + h := c.Get(uint64(ns), key, func() (int, Value) { + o := &objects[key] + o.acquire() + return 1, o + }) + if v := h.Value().(*int32o); v != &objects[key] { + t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v) + } + if objects[key] != 1 { + t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key]) + } + if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) { + h.Release() + } + } + }(ns, i, x.repeat, objects[ns], handles[ns]) + } + + go func(handles []unsafe.Pointer) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for atomic.LoadInt32(&done) == 0 { + i := r.Intn(len(handles)) + h := (*Handle)(atomic.LoadPointer(&handles[i])) + if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) { + h.Release() + } + time.Sleep(time.Millisecond) + } + }(handles[ns]) + } + + go func() { + handles := make([]*Handle, 100000) + for atomic.LoadInt32(&done) == 0 { + for i := range handles { + handles[i] = c.Get(999999999, uint64(i), func() (int, Value) { + return 1, 1 + }) + } + for _, h := range handles { + h.Release() + } + } + }() + + wg.Wait() + + atomic.StoreInt32(&done, 1) + + for _, handles0 := range handles { + for i := range handles0 { + h := (*Handle)(atomic.LoadPointer(&handles0[i])) + if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) { + h.Release() + } + } + } + + for ns, objects0 := range objects { + for i, o := range objects0 { + if o != 0 { + t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o) + } + } + } +} + +func TestCacheMap_NodesAndSize(t *testing.T) { + c := NewCache(nil) + if c.Nodes() != 0 { + t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) + } + if c.Size() != 0 { + t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) + } + set(c, 0, 1, 1, 1, nil) + set(c, 0, 2, 2, 2, nil) + set(c, 1, 1, 3, 3, nil) + set(c, 2, 1, 4, 1, nil) + if c.Nodes() != 4 { + t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes()) + } + if c.Size() != 7 { + t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size()) + } +} + +func TestLRUCache_Capacity(t *testing.T) { + c := NewCache(NewLRU(10)) + if c.Capacity() != 10 { + t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity()) + } + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 2, nil).Release() + set(c, 1, 1, 3, 3, nil).Release() + set(c, 2, 1, 4, 1, nil).Release() + set(c, 2, 2, 5, 1, nil).Release() + set(c, 2, 3, 6, 1, nil).Release() + set(c, 2, 4, 7, 1, nil).Release() + set(c, 2, 5, 8, 1, nil).Release() + if c.Nodes() != 7 { + t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes()) + } + if c.Size() != 10 { + t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size()) + } + c.SetCapacity(9) + if c.Capacity() != 9 { + t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity()) + } + if c.Nodes() != 6 { + t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes()) + } + if c.Size() != 8 { + t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size()) + } +} + +func TestCacheMap_NilValue(t *testing.T) { + c := NewCache(NewLRU(10)) + h := c.Get(0, 0, func() (size int, value Value) { + return 1, nil + }) + if h != nil { + t.Error("cache handle is non-nil") + } + if c.Nodes() != 0 { + t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) + } + if c.Size() != 0 { + t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) + } +} + +func TestLRUCache_GetLatency(t *testing.T) { + runtime.GOMAXPROCS(runtime.NumCPU()) + + const ( + concurrentSet = 30 + concurrentGet = 3 + duration = 3 * time.Second + delay = 3 * time.Millisecond + maxkey = 100000 + ) + + var ( + set, getHit, getAll int32 + getMaxLatency, getDuration int64 + ) + + c := NewCache(NewLRU(5000)) + wg := &sync.WaitGroup{} + until := time.Now().Add(duration) + for i := 0; i < concurrentSet; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for time.Now().Before(until) { + c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) { + time.Sleep(delay) + atomic.AddInt32(&set, 1) + return 1, 1 + }).Release() + } + }(i) + } + for i := 0; i < concurrentGet; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for { + mark := time.Now() + if mark.Before(until) { + h := c.Get(0, uint64(r.Intn(maxkey)), nil) + latency := int64(time.Now().Sub(mark)) + m := atomic.LoadInt64(&getMaxLatency) + if latency > m { + atomic.CompareAndSwapInt64(&getMaxLatency, m, latency) + } + atomic.AddInt64(&getDuration, latency) + if h != nil { + atomic.AddInt32(&getHit, 1) + h.Release() + } + atomic.AddInt32(&getAll, 1) + } else { + break + } + } + }(i) + } + + wg.Wait() + getAvglatency := time.Duration(getDuration) / time.Duration(getAll) + t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v", + set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency) + + if getAvglatency > delay/3 { + t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency) + } +} + +func TestLRUCache_HitMiss(t *testing.T) { cases := []struct { key uint64 value string @@ -54,14 +308,13 @@ func TestCache_HitMiss(t *testing.T) { } setfin := 0 - c := NewLRUCache(1000) - ns := c.GetNamespace(0) + c := NewCache(NewLRU(1000)) for i, x := range cases { - set(ns, x.key, x.value, len(x.value), func() { + set(c, 0, x.key, x.value, len(x.value), func() { setfin++ }).Release() for j, y := range cases { - h := ns.Get(y.key, nil) + h := c.Get(0, y.key, nil) if j <= i { // should hit if h == nil { @@ -85,7 +338,7 @@ func TestCache_HitMiss(t *testing.T) { for i, x := range cases { finalizerOk := false - ns.Delete(x.key, func(exist, pending bool) { + c.Delete(0, x.key, func() { finalizerOk = true }) @@ -94,7 +347,7 @@ func TestCache_HitMiss(t *testing.T) { } for j, y := range cases { - h := ns.Get(y.key, nil) + h := c.Get(0, y.key, nil) if j > i { // should hit if h == nil { @@ -122,20 +375,19 @@ func TestCache_HitMiss(t *testing.T) { } func TestLRUCache_Eviction(t *testing.T) { - c := NewLRUCache(12) - ns := c.GetNamespace(0) - o1 := set(ns, 1, 1, 1, nil) - set(ns, 2, 2, 1, nil).Release() - set(ns, 3, 3, 1, nil).Release() - set(ns, 4, 4, 1, nil).Release() - set(ns, 5, 5, 1, nil).Release() - if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2 + c := NewCache(NewLRU(12)) + o1 := set(c, 0, 1, 1, 1, nil) + set(c, 0, 2, 2, 1, nil).Release() + set(c, 0, 3, 3, 1, nil).Release() + set(c, 0, 4, 4, 1, nil).Release() + set(c, 0, 5, 5, 1, nil).Release() + if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2 h.Release() } - set(ns, 9, 9, 10, nil).Release() // 5,2,9 + set(c, 0, 9, 9, 10, nil).Release() // 5,2,9 for _, key := range []uint64{9, 2, 5, 1} { - h := ns.Get(key, nil) + h := c.Get(0, key, nil) if h == nil { t.Errorf("miss for key '%d'", key) } else { @@ -147,7 +399,7 @@ func TestLRUCache_Eviction(t *testing.T) { } o1.Release() for _, key := range []uint64{1, 2, 5} { - h := ns.Get(key, nil) + h := c.Get(0, key, nil) if h == nil { t.Errorf("miss for key '%d'", key) } else { @@ -158,7 +410,7 @@ func TestLRUCache_Eviction(t *testing.T) { } } for _, key := range []uint64{3, 4, 9} { - h := ns.Get(key, nil) + h := c.Get(0, key, nil) if h != nil { t.Errorf("hit for key '%d'", key) if x := h.Value().(int); x != int(key) { @@ -169,487 +421,134 @@ func TestLRUCache_Eviction(t *testing.T) { } } -func TestLRUCache_SetGet(t *testing.T) { - c := NewLRUCache(13) - ns := c.GetNamespace(0) - for i := 0; i < 200; i++ { - n := uint64(rand.Intn(99999) % 20) - set(ns, n, n, 1, nil).Release() - if h := ns.Get(n, nil); h != nil { - if h.Value() == nil { - t.Errorf("key '%d' contains nil value", n) +func TestLRUCache_Evict(t *testing.T) { + c := NewCache(NewLRU(6)) + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 1, nil).Release() + set(c, 1, 1, 4, 1, nil).Release() + set(c, 1, 2, 5, 1, nil).Release() + set(c, 2, 1, 6, 1, nil).Release() + set(c, 2, 2, 7, 1, nil).Release() + + for ns := 0; ns < 3; ns++ { + for key := 1; key < 3; key++ { + if h := c.Get(uint64(ns), uint64(key), nil); h != nil { + h.Release() } else { - if x := h.Value().(uint64); x != n { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x) - } + t.Errorf("Cache.Get on #%d.%d return nil", ns, key) } + } + } + + if ok := c.Evict(0, 1); !ok { + t.Error("first Cache.Evict on #0.1 return false") + } + if ok := c.Evict(0, 1); ok { + t.Error("second Cache.Evict on #0.1 return true") + } + if h := c.Get(0, 1, nil); h != nil { + t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value()) + } + + c.EvictNS(1) + if h := c.Get(1, 1, nil); h != nil { + t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value()) + } + if h := c.Get(1, 2, nil); h != nil { + t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value()) + } + + c.EvictAll() + for ns := 0; ns < 3; ns++ { + for key := 1; key < 3; key++ { + if h := c.Get(uint64(ns), uint64(key), nil); h != nil { + t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value()) + } + } + } +} + +func TestLRUCache_Delete(t *testing.T) { + delFuncCalled := 0 + delFunc := func() { + delFuncCalled++ + } + + c := NewCache(NewLRU(2)) + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 1, nil).Release() + + if ok := c.Delete(0, 1, delFunc); !ok { + t.Error("Cache.Delete on #1 return false") + } + if h := c.Get(0, 1, nil); h != nil { + t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value()) + } + if ok := c.Delete(0, 1, delFunc); ok { + t.Error("Cache.Delete on #1 return true") + } + + h2 := c.Get(0, 2, nil) + if h2 == nil { + t.Error("Cache.Get on #2 return nil") + } + if ok := c.Delete(0, 2, delFunc); !ok { + t.Error("(1) Cache.Delete on #2 return false") + } + if ok := c.Delete(0, 2, delFunc); !ok { + t.Error("(2) Cache.Delete on #2 return false") + } + + set(c, 0, 3, 3, 1, nil).Release() + set(c, 0, 4, 4, 1, nil).Release() + c.Get(0, 2, nil).Release() + + for key := 2; key <= 4; key++ { + if h := c.Get(0, uint64(key), nil); h != nil { h.Release() } else { - t.Errorf("key '%d' doesn't exist", n) - } - } -} - -func TestLRUCache_Purge(t *testing.T) { - c := NewLRUCache(3) - ns1 := c.GetNamespace(0) - o1 := set(ns1, 1, 1, 1, nil) - o2 := set(ns1, 2, 2, 1, nil) - ns1.Purge(nil) - set(ns1, 3, 3, 1, nil).Release() - for _, key := range []uint64{1, 2, 3} { - h := ns1.Get(key, nil) - if h == nil { - t.Errorf("miss for key '%d'", key) - } else { - if x := h.Value().(int); x != int(key) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) - } - h.Release() - } - } - o1.Release() - o2.Release() - for _, key := range []uint64{1, 2} { - h := ns1.Get(key, nil) - if h != nil { - t.Errorf("hit for key '%d'", key) - if x := h.Value().(int); x != int(key) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) - } - h.Release() - } - } -} - -type testingCacheObjectCounter struct { - created uint - released uint -} - -func (c *testingCacheObjectCounter) createOne() { - c.created++ -} - -func (c *testingCacheObjectCounter) releaseOne() { - c.released++ -} - -type testingCacheObject struct { - t *testing.T - cnt *testingCacheObjectCounter - - ns, key uint64 - - releaseCalled bool -} - -func (x *testingCacheObject) Release() { - if !x.releaseCalled { - x.releaseCalled = true - x.cnt.releaseOne() - } else { - x.t.Errorf("duplicate setfin NS#%d KEY#%d", x.ns, x.key) - } -} - -func TestLRUCache_ConcurrentSetGet(t *testing.T) { - runtime.GOMAXPROCS(runtime.NumCPU()) - - seed := time.Now().UnixNano() - t.Logf("seed=%d", seed) - - const ( - N = 2000000 - M = 4000 - C = 3 - ) - - var set, get uint32 - - wg := &sync.WaitGroup{} - c := NewLRUCache(M / 4) - for ni := uint64(0); ni < C; ni++ { - r0 := rand.New(rand.NewSource(seed + int64(ni))) - r1 := rand.New(rand.NewSource(seed + int64(ni) + 1)) - ns := c.GetNamespace(ni) - - wg.Add(2) - go func(ns Namespace, r *rand.Rand) { - for i := 0; i < N; i++ { - x := uint64(r.Int63n(M)) - o := ns.Get(x, func() (int, interface{}) { - atomic.AddUint32(&set, 1) - return 1, x - }) - if v := o.Value().(uint64); v != x { - t.Errorf("#%d invalid value, got=%d", x, v) - } - o.Release() - } - wg.Done() - }(ns, r0) - go func(ns Namespace, r *rand.Rand) { - for i := 0; i < N; i++ { - x := uint64(r.Int63n(M)) - o := ns.Get(x, nil) - if o != nil { - atomic.AddUint32(&get, 1) - if v := o.Value().(uint64); v != x { - t.Errorf("#%d invalid value, got=%d", x, v) - } - o.Release() - } - } - wg.Done() - }(ns, r1) - } - - wg.Wait() - - t.Logf("set=%d get=%d", set, get) -} - -func TestLRUCache_Finalizer(t *testing.T) { - const ( - capacity = 100 - goroutines = 100 - iterations = 10000 - keymax = 8000 - ) - - cnt := &testingCacheObjectCounter{} - - c := NewLRUCache(capacity) - - type instance struct { - seed int64 - rnd *rand.Rand - nsid uint64 - ns Namespace - effective int - handles []Handle - handlesMap map[uint64]int - - delete bool - purge bool - zap bool - wantDel int - delfinCalled int - delfinCalledAll int - delfinCalledEff int - purgefinCalled int - } - - instanceGet := func(p *instance, key uint64) { - h := p.ns.Get(key, func() (charge int, value interface{}) { - to := &testingCacheObject{ - t: t, cnt: cnt, - ns: p.nsid, - key: key, - } - p.effective++ - cnt.createOne() - return 1, releaserFunc{func() { - to.Release() - p.effective-- - }, to} - }) - p.handles = append(p.handles, h) - p.handlesMap[key] = p.handlesMap[key] + 1 - } - instanceRelease := func(p *instance, i int) { - h := p.handles[i] - key := h.Value().(releaserFunc).value.(*testingCacheObject).key - if n := p.handlesMap[key]; n == 0 { - t.Fatal("key ref == 0") - } else if n > 1 { - p.handlesMap[key] = n - 1 - } else { - delete(p.handlesMap, key) - } - h.Release() - p.handles = append(p.handles[:i], p.handles[i+1:]...) - p.handles[len(p.handles) : len(p.handles)+1][0] = nil - } - - seed := time.Now().UnixNano() - t.Logf("seed=%d", seed) - - instances := make([]*instance, goroutines) - for i := range instances { - p := &instance{} - p.handlesMap = make(map[uint64]int) - p.seed = seed + int64(i) - p.rnd = rand.New(rand.NewSource(p.seed)) - p.nsid = uint64(i) - p.ns = c.GetNamespace(p.nsid) - p.delete = i%6 == 0 - p.purge = i%8 == 0 - p.zap = i%12 == 0 || i%3 == 0 - instances[i] = p - } - - runr := rand.New(rand.NewSource(seed - 1)) - run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) { - var ( - rx []*instance - rn []int - ) - if init == nil { - rx = append([]*instance{}, x...) - rn = make([]int, len(x)) - } else { - for _, p := range x { - if init(p) { - rx = append(rx, p) - rn = append(rn, 0) - } - } - } - for len(rx) > 0 { - i := rand.Intn(len(rx)) - if fn(rx[i], rn[i]) { - rn[i]++ - } else { - rx = append(rx[:i], rx[i+1:]...) - rn = append(rn[:i], rn[i+1:]...) - } + t.Errorf("Cache.Get on #%d return nil", key) } } - // Get and release. - run(runr, instances, nil, func(p *instance, i int) bool { - if i < iterations { - if len(p.handles) == 0 || p.rnd.Int()%2 == 0 { - instanceGet(p, uint64(p.rnd.Intn(keymax))) - } else { - instanceRelease(p, p.rnd.Intn(len(p.handles))) - } - return true - } else { - return false - } - }) - - if used, cap := c.Used(), c.Capacity(); used > cap { - t.Errorf("Used > capacity, used=%d cap=%d", used, cap) + h2.Release() + if h := c.Get(0, 2, nil); h != nil { + t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value()) } - // Check effective objects. - for i, p := range instances { - if int(p.effective) < len(p.handlesMap) { - t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap)) - } - } - - if want := int(cnt.created - cnt.released); c.Size() != want { - t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size()) - } - - // First delete. - run(runr, instances, func(p *instance) bool { - p.wantDel = p.effective - return p.delete - }, func(p *instance, i int) bool { - key := uint64(i) - if key < keymax { - _, wantExist := p.handlesMap[key] - gotExist := p.ns.Delete(key, func(exist, pending bool) { - p.delfinCalledAll++ - if exist { - p.delfinCalledEff++ - } - }) - if !gotExist && wantExist { - t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key) - } - return true - } else { - return false - } - }) - - // Second delete. - run(runr, instances, func(p *instance) bool { - p.delfinCalled = 0 - return p.delete - }, func(p *instance, i int) bool { - key := uint64(i) - if key < keymax { - gotExist := p.ns.Delete(key, func(exist, pending bool) { - if exist && !pending { - t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key) - } - p.delfinCalled++ - }) - if gotExist { - t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key) - } - return true - } else { - if p.delfinCalled != keymax { - t.Errorf("(2) NS#%d not all delete fin called, diff=%d", p.nsid, keymax-p.delfinCalled) - } - return false - } - }) - - // Purge. - run(runr, instances, func(p *instance) bool { - return p.purge - }, func(p *instance, i int) bool { - p.ns.Purge(func(ns, key uint64) { - p.purgefinCalled++ - }) - return false - }) - - if want := int(cnt.created - cnt.released); c.Size() != want { - t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size()) - } - - // Release. - run(runr, instances, func(p *instance) bool { - return !p.zap - }, func(p *instance, i int) bool { - if len(p.handles) > 0 { - instanceRelease(p, len(p.handles)-1) - return true - } else { - return false - } - }) - - if want := int(cnt.created - cnt.released); c.Size() != want { - t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size()) - } - - // Zap. - run(runr, instances, func(p *instance) bool { - return p.zap - }, func(p *instance, i int) bool { - p.ns.Zap() - p.handles = nil - p.handlesMap = nil - return false - }) - - if want := int(cnt.created - cnt.released); c.Size() != want { - t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size()) - } - - if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used { - t.Errorf("Invalid used value, want=%d got=%d", notrel, used) - } - - c.Purge(nil) - - for _, p := range instances { - if p.delete { - if p.delfinCalledAll != keymax { - t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll) - } - if p.delfinCalledEff != p.wantDel { - t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff) - } - if p.purge && p.purgefinCalled > 0 { - t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled) - } - } else { - if p.purge { - if p.purgefinCalled != p.wantDel { - t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled) - } - } - } - } - - if cnt.created != cnt.released { - t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released) + if delFuncCalled != 4 { + t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled) } } -func BenchmarkLRUCache_Set(b *testing.B) { - c := NewLRUCache(0) - ns := c.GetNamespace(0) - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - set(ns, i, "", 1, nil) - } -} - -func BenchmarkLRUCache_Get(b *testing.B) { - c := NewLRUCache(0) - ns := c.GetNamespace(0) - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - set(ns, i, "", 1, nil) - } - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - ns.Get(i, nil) - } -} - -func BenchmarkLRUCache_Get2(b *testing.B) { - c := NewLRUCache(0) - ns := c.GetNamespace(0) - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - set(ns, i, "", 1, nil) - } - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - ns.Get(i, func() (charge int, value interface{}) { - return 0, nil - }) - } -} - -func BenchmarkLRUCache_Release(b *testing.B) { - c := NewLRUCache(0) - ns := c.GetNamespace(0) - handles := make([]Handle, b.N) - for i := uint64(0); i < uint64(b.N); i++ { - handles[i] = set(ns, i, "", 1, nil) - } - b.ResetTimer() - for _, h := range handles { - h.Release() - } -} - -func BenchmarkLRUCache_SetRelease(b *testing.B) { - capacity := b.N / 100 - if capacity <= 0 { - capacity = 10 - } - c := NewLRUCache(capacity) - ns := c.GetNamespace(0) - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - set(ns, i, "", 1, nil).Release() - } -} - -func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) { - capacity := b.N / 100 - if capacity <= 0 { - capacity = 10 - } - c := NewLRUCache(capacity) - ns := c.GetNamespace(0) - b.ResetTimer() - - na := b.N / 2 - nb := b.N - na - - for i := uint64(0); i < uint64(na); i++ { - set(ns, i, "", 1, nil).Release() - } - - for i := uint64(0); i < uint64(nb); i++ { - set(ns, i, "", 1, nil).Release() +func TestLRUCache_Close(t *testing.T) { + relFuncCalled := 0 + relFunc := func() { + relFuncCalled++ + } + delFuncCalled := 0 + delFunc := func() { + delFuncCalled++ + } + + c := NewCache(NewLRU(2)) + set(c, 0, 1, 1, 1, relFunc).Release() + set(c, 0, 2, 2, 1, relFunc).Release() + + h3 := set(c, 0, 3, 3, 1, relFunc) + if h3 == nil { + t.Error("Cache.Get on #3 return nil") + } + if ok := c.Delete(0, 3, delFunc); !ok { + t.Error("Cache.Delete on #3 return false") + } + + c.Close() + + if relFuncCalled != 3 { + t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled) + } + if delFuncCalled != 1 { + t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled) } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go new file mode 100644 index 000000000..d9a84cde1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go @@ -0,0 +1,195 @@ +// Copyright (c) 2012, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package cache + +import ( + "sync" + "unsafe" +) + +type lruNode struct { + n *Node + h *Handle + ban bool + + next, prev *lruNode +} + +func (n *lruNode) insert(at *lruNode) { + x := at.next + at.next = n + n.prev = at + n.next = x + x.prev = n +} + +func (n *lruNode) remove() { + if n.prev != nil { + n.prev.next = n.next + n.next.prev = n.prev + n.prev = nil + n.next = nil + } else { + panic("BUG: removing removed node") + } +} + +type lru struct { + mu sync.Mutex + capacity int + used int + recent lruNode +} + +func (r *lru) reset() { + r.recent.next = &r.recent + r.recent.prev = &r.recent + r.used = 0 +} + +func (r *lru) Capacity() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.capacity +} + +func (r *lru) SetCapacity(capacity int) { + var evicted []*lruNode + + r.mu.Lock() + r.capacity = capacity + for r.used > r.capacity { + rn := r.recent.prev + if rn == nil { + panic("BUG: invalid LRU used or capacity counter") + } + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) Promote(n *Node) { + var evicted []*lruNode + + r.mu.Lock() + if n.CacheData == nil { + if n.Size() <= r.capacity { + rn := &lruNode{n: n, h: n.GetHandle()} + rn.insert(&r.recent) + n.CacheData = unsafe.Pointer(rn) + r.used += n.Size() + + for r.used > r.capacity { + rn := r.recent.prev + if rn == nil { + panic("BUG: invalid LRU used or capacity counter") + } + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + } + } else { + rn := (*lruNode)(n.CacheData) + if !rn.ban { + rn.remove() + rn.insert(&r.recent) + } + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) Ban(n *Node) { + r.mu.Lock() + if n.CacheData == nil { + n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) + } else { + rn := (*lruNode)(n.CacheData) + if !rn.ban { + rn.remove() + rn.ban = true + r.used -= rn.n.Size() + r.mu.Unlock() + + rn.h.Release() + rn.h = nil + return + } + } + r.mu.Unlock() +} + +func (r *lru) Evict(n *Node) { + r.mu.Lock() + rn := (*lruNode)(n.CacheData) + if rn == nil || rn.ban { + r.mu.Unlock() + return + } + n.CacheData = nil + r.mu.Unlock() + + rn.h.Release() +} + +func (r *lru) EvictNS(ns uint64) { + var evicted []*lruNode + + r.mu.Lock() + for e := r.recent.prev; e != &r.recent; { + rn := e + e = e.prev + if rn.n.NS() == ns { + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) EvictAll() { + r.mu.Lock() + back := r.recent.prev + for rn := back; rn != &r.recent; rn = rn.prev { + rn.n.CacheData = nil + } + r.reset() + r.mu.Unlock() + + for rn := back; rn != &r.recent; rn = rn.prev { + rn.h.Release() + } +} + +func (r *lru) Close() error { + return nil +} + +// NewLRU create a new LRU-cache. +func NewLRU(capacity int) Cacher { + r := &lru{capacity: capacity} + r.reset() + return r +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go deleted file mode 100644 index 853676cc4..000000000 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go +++ /dev/null @@ -1,622 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "sync" - "sync/atomic" - - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util" -) - -// The LLRB implementation were taken from https://github.com/petar/GoLLRB. -// Which contains the following header: -// -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// lruCache represent a LRU cache state. -type lruCache struct { - mu sync.Mutex - recent lruNode - table map[uint64]*lruNs - capacity int - used, size, alive int -} - -// NewLRUCache creates a new initialized LRU cache with the given capacity. -func NewLRUCache(capacity int) Cache { - c := &lruCache{ - table: make(map[uint64]*lruNs), - capacity: capacity, - } - c.recent.rNext = &c.recent - c.recent.rPrev = &c.recent - return c -} - -func (c *lruCache) Capacity() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.capacity -} - -func (c *lruCache) Used() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.used -} - -func (c *lruCache) Size() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.size -} - -func (c *lruCache) NumObjects() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.alive -} - -// SetCapacity set cache capacity. -func (c *lruCache) SetCapacity(capacity int) { - c.mu.Lock() - c.capacity = capacity - c.evict() - c.mu.Unlock() -} - -// GetNamespace return namespace object for given id. -func (c *lruCache) GetNamespace(id uint64) Namespace { - c.mu.Lock() - defer c.mu.Unlock() - - if ns, ok := c.table[id]; ok { - return ns - } - - ns := &lruNs{lru: c, id: id} - c.table[id] = ns - return ns -} - -func (c *lruCache) ZapNamespace(id uint64) { - c.mu.Lock() - if ns, exist := c.table[id]; exist { - ns.zapNB() - delete(c.table, id) - } - c.mu.Unlock() -} - -func (c *lruCache) PurgeNamespace(id uint64, fin PurgeFin) { - c.mu.Lock() - if ns, exist := c.table[id]; exist { - ns.purgeNB(fin) - } - c.mu.Unlock() -} - -// Purge purge entire cache. -func (c *lruCache) Purge(fin PurgeFin) { - c.mu.Lock() - for _, ns := range c.table { - ns.purgeNB(fin) - } - c.mu.Unlock() -} - -func (c *lruCache) Zap() { - c.mu.Lock() - for _, ns := range c.table { - ns.zapNB() - } - c.table = make(map[uint64]*lruNs) - c.mu.Unlock() -} - -func (c *lruCache) evict() { - top := &c.recent - for n := c.recent.rPrev; c.used > c.capacity && n != top; { - if n.state != nodeEffective { - panic("evicting non effective node") - } - n.state = nodeEvicted - n.rRemove() - n.derefNB() - c.used -= n.charge - n = c.recent.rPrev - } -} - -type lruNs struct { - lru *lruCache - id uint64 - rbRoot *lruNode - state nsState -} - -func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) { - if h == nil { - n = &lruNode{ns: ns, key: key} - return n, n - } - - if key < h.key { - hn, n = ns.rbGetOrCreateNode(h.rbLeft, key) - if hn != nil { - h.rbLeft = hn - } else { - return nil, n - } - } else if key > h.key { - hn, n = ns.rbGetOrCreateNode(h.rbRight, key) - if hn != nil { - h.rbRight = hn - } else { - return nil, n - } - } else { - return nil, h - } - - if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) { - h = rbRotLeft(h) - } - if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) { - h = rbRotRight(h) - } - if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) { - rbFlip(h) - } - return h, n -} - -func (ns *lruNs) getOrCreateNode(key uint64) *lruNode { - hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key) - if hn != nil { - ns.rbRoot = hn - ns.rbRoot.rbBlack = true - } - return n -} - -func (ns *lruNs) rbGetNode(key uint64) *lruNode { - h := ns.rbRoot - for h != nil { - switch { - case key < h.key: - h = h.rbLeft - case key > h.key: - h = h.rbRight - default: - return h - } - } - return nil -} - -func (ns *lruNs) getNode(key uint64) *lruNode { - return ns.rbGetNode(key) -} - -func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode { - if h == nil { - return nil - } - - if key < h.key { - if h.rbLeft == nil { // key not present. Nothing to delete - return h - } - if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) { - h = rbMoveLeft(h) - } - h.rbLeft = ns.rbDeleteNode(h.rbLeft, key) - } else { - if rbIsRed(h.rbLeft) { - h = rbRotRight(h) - } - // If @key equals @h.key and no right children at @h - if h.key == key && h.rbRight == nil { - return nil - } - if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) { - h = rbMoveRight(h) - } - // If @key equals @h.key, and (from above) 'h.Right != nil' - if h.key == key { - var x *lruNode - h.rbRight, x = rbDeleteMin(h.rbRight) - if x == nil { - panic("logic") - } - x.rbLeft, h.rbLeft = h.rbLeft, nil - x.rbRight, h.rbRight = h.rbRight, nil - x.rbBlack = h.rbBlack - h = x - } else { // Else, @key is bigger than @h.key - h.rbRight = ns.rbDeleteNode(h.rbRight, key) - } - } - - return rbFixup(h) -} - -func (ns *lruNs) deleteNode(key uint64) { - ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key) - if ns.rbRoot != nil { - ns.rbRoot.rbBlack = true - } -} - -func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool { - if h == nil { - return true - } - if h.key >= pivot { - if !ns.rbIterateNodes(h.rbLeft, pivot, iter) { - return false - } - if !iter(h) { - return false - } - } - return ns.rbIterateNodes(h.rbRight, pivot, iter) -} - -func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) { - ns.rbIterateNodes(ns.rbRoot, 0, iter) -} - -func (ns *lruNs) Get(key uint64, setf SetFunc) Handle { - ns.lru.mu.Lock() - defer ns.lru.mu.Unlock() - - if ns.state != nsEffective { - return nil - } - - var n *lruNode - if setf == nil { - n = ns.getNode(key) - if n == nil { - return nil - } - } else { - n = ns.getOrCreateNode(key) - } - switch n.state { - case nodeZero: - charge, value := setf() - if value == nil { - ns.deleteNode(key) - return nil - } - if charge < 0 { - charge = 0 - } - - n.value = value - n.charge = charge - n.state = nodeEvicted - - ns.lru.size += charge - ns.lru.alive++ - - fallthrough - case nodeEvicted: - if n.charge == 0 { - break - } - - // Insert to recent list. - n.state = nodeEffective - n.ref++ - ns.lru.used += n.charge - ns.lru.evict() - - fallthrough - case nodeEffective: - // Bump to front. - n.rRemove() - n.rInsert(&ns.lru.recent) - case nodeDeleted: - // Do nothing. - default: - panic("invalid state") - } - n.ref++ - - return &lruHandle{node: n} -} - -func (ns *lruNs) Delete(key uint64, fin DelFin) bool { - ns.lru.mu.Lock() - defer ns.lru.mu.Unlock() - - if ns.state != nsEffective { - if fin != nil { - fin(false, false) - } - return false - } - - n := ns.getNode(key) - if n == nil { - if fin != nil { - fin(false, false) - } - return false - - } - - switch n.state { - case nodeEffective: - ns.lru.used -= n.charge - n.state = nodeDeleted - n.delfin = fin - n.rRemove() - n.derefNB() - case nodeEvicted: - n.state = nodeDeleted - n.delfin = fin - case nodeDeleted: - if fin != nil { - fin(true, true) - } - return false - default: - panic("invalid state") - } - - return true -} - -func (ns *lruNs) purgeNB(fin PurgeFin) { - if ns.state == nsEffective { - var nodes []*lruNode - ns.iterateNodes(func(n *lruNode) bool { - nodes = append(nodes, n) - return true - }) - for _, n := range nodes { - switch n.state { - case nodeEffective: - ns.lru.used -= n.charge - n.state = nodeDeleted - n.purgefin = fin - n.rRemove() - n.derefNB() - case nodeEvicted: - n.state = nodeDeleted - n.purgefin = fin - case nodeDeleted: - default: - panic("invalid state") - } - } - } -} - -func (ns *lruNs) Purge(fin PurgeFin) { - ns.lru.mu.Lock() - ns.purgeNB(fin) - ns.lru.mu.Unlock() -} - -func (ns *lruNs) zapNB() { - if ns.state == nsEffective { - ns.state = nsZapped - - ns.iterateNodes(func(n *lruNode) bool { - if n.state == nodeEffective { - ns.lru.used -= n.charge - n.rRemove() - } - ns.lru.size -= n.charge - n.state = nodeDeleted - n.fin() - - return true - }) - ns.rbRoot = nil - } -} - -func (ns *lruNs) Zap() { - ns.lru.mu.Lock() - ns.zapNB() - delete(ns.lru.table, ns.id) - ns.lru.mu.Unlock() -} - -type lruNode struct { - ns *lruNs - - rNext, rPrev *lruNode - rbLeft, rbRight *lruNode - rbBlack bool - - key uint64 - value interface{} - charge int - ref int - state nodeState - delfin DelFin - purgefin PurgeFin -} - -func (n *lruNode) rInsert(at *lruNode) { - x := at.rNext - at.rNext = n - n.rPrev = at - n.rNext = x - x.rPrev = n -} - -func (n *lruNode) rRemove() bool { - if n.rPrev == nil { - return false - } - - n.rPrev.rNext = n.rNext - n.rNext.rPrev = n.rPrev - n.rPrev = nil - n.rNext = nil - - return true -} - -func (n *lruNode) fin() { - if r, ok := n.value.(util.Releaser); ok { - r.Release() - } - if n.purgefin != nil { - if n.delfin != nil { - panic("conflicting delete and purge fin") - } - n.purgefin(n.ns.id, n.key) - n.purgefin = nil - } else if n.delfin != nil { - n.delfin(true, false) - n.delfin = nil - } -} - -func (n *lruNode) derefNB() { - n.ref-- - if n.ref == 0 { - if n.ns.state == nsEffective { - // Remove elemement. - n.ns.deleteNode(n.key) - n.ns.lru.size -= n.charge - n.ns.lru.alive-- - n.fin() - } - n.value = nil - } else if n.ref < 0 { - panic("leveldb/cache: lruCache: negative node reference") - } -} - -func (n *lruNode) deref() { - n.ns.lru.mu.Lock() - n.derefNB() - n.ns.lru.mu.Unlock() -} - -type lruHandle struct { - node *lruNode - once uint32 -} - -func (h *lruHandle) Value() interface{} { - if atomic.LoadUint32(&h.once) == 0 { - return h.node.value - } - return nil -} - -func (h *lruHandle) Release() { - if !atomic.CompareAndSwapUint32(&h.once, 0, 1) { - return - } - h.node.deref() - h.node = nil -} - -func rbIsRed(h *lruNode) bool { - if h == nil { - return false - } - return !h.rbBlack -} - -func rbRotLeft(h *lruNode) *lruNode { - x := h.rbRight - if x.rbBlack { - panic("rotating a black link") - } - h.rbRight = x.rbLeft - x.rbLeft = h - x.rbBlack = h.rbBlack - h.rbBlack = false - return x -} - -func rbRotRight(h *lruNode) *lruNode { - x := h.rbLeft - if x.rbBlack { - panic("rotating a black link") - } - h.rbLeft = x.rbRight - x.rbRight = h - x.rbBlack = h.rbBlack - h.rbBlack = false - return x -} - -func rbFlip(h *lruNode) { - h.rbBlack = !h.rbBlack - h.rbLeft.rbBlack = !h.rbLeft.rbBlack - h.rbRight.rbBlack = !h.rbRight.rbBlack -} - -func rbMoveLeft(h *lruNode) *lruNode { - rbFlip(h) - if rbIsRed(h.rbRight.rbLeft) { - h.rbRight = rbRotRight(h.rbRight) - h = rbRotLeft(h) - rbFlip(h) - } - return h -} - -func rbMoveRight(h *lruNode) *lruNode { - rbFlip(h) - if rbIsRed(h.rbLeft.rbLeft) { - h = rbRotRight(h) - rbFlip(h) - } - return h -} - -func rbFixup(h *lruNode) *lruNode { - if rbIsRed(h.rbRight) { - h = rbRotLeft(h) - } - - if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) { - h = rbRotRight(h) - } - - if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) { - rbFlip(h) - } - - return h -} - -func rbDeleteMin(h *lruNode) (hn, n *lruNode) { - if h == nil { - return nil, nil - } - if h.rbLeft == nil { - return nil, h - } - - if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) { - h = rbMoveLeft(h) - } - - h.rbLeft, n = rbDeleteMin(h.rbLeft) - - return rbFixup(h), n -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go index 336549896..94e002841 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go @@ -9,14 +9,12 @@ package leveldb import ( "bytes" "fmt" - "io" - "math/rand" - "testing" - - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage" + "io" + "math/rand" + "testing" ) const ctValSize = 1000 @@ -33,8 +31,8 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness { func newDbCorruptHarness(t *testing.T) *dbCorruptHarness { return newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCache: cache.NewLRUCache(100), - Strict: opt.StrictJournalChecksum, + BlockCacheCapacity: 100, + Strict: opt.StrictJournalChecksum, }) } @@ -269,9 +267,9 @@ func TestCorruptDB_TableIndex(t *testing.T) { func TestCorruptDB_MissingManifest(t *testing.T) { rnd := rand.New(rand.NewSource(0x0badda7a)) h := newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCache: cache.NewLRUCache(100), - Strict: opt.StrictJournalChecksum, - WriteBuffer: 1000 * 60, + BlockCacheCapacity: 100, + Strict: opt.StrictJournalChecksum, + WriteBuffer: 1000 * 60, }) h.build(1000) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go index 1d5e73d2b..219549767 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go @@ -27,8 +27,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util" ) -// DB is a LevelDB database. -type DB struct { +type DB struct { // DB is a LevelDB database. // Need 64-bit alignment. seq uint64 @@ -347,12 +346,14 @@ func recoverTable(s *session, o *opt.Options) error { return err } iter := tr.NewIterator(nil, nil) - iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) { - if errors.IsCorrupted(err) { - s.logf("table@recovery block corruption @%d %q", file.Num(), err) - tcorruptedBlock++ - } - }) + if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok { + itererr.SetErrorCallback(func(err error) { + if errors.IsCorrupted(err) { + s.logf("table@recovery block corruption @%d %q", file.Num(), err) + tcorruptedBlock++ + } + }) + } // Scan the table. for iter.Next() { @@ -823,8 +824,8 @@ func (db *DB) GetProperty(name string) (value string, err error) { case p == "blockpool": value = fmt.Sprintf("%v", db.s.tops.bpool) case p == "cachedblock": - if bc := db.s.o.GetBlockCache(); bc != nil { - value = fmt.Sprintf("%d", bc.Size()) + if db.s.tops.bcache != nil { + value = fmt.Sprintf("%d", db.s.tops.bcache.Size()) } else { value = "" } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go index 9c83497f5..f59f92bad 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go @@ -1271,7 +1271,7 @@ func TestDB_DeletionMarkers2(t *testing.T) { } func TestDB_CompactionTableOpenError(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{CachedOpenFiles: -1}) + h := newDbHarnessWopt(t, &opt.Options{OpenFilesCacheCapacity: -1}) defer h.close() im := 10 @@ -1629,8 +1629,8 @@ func TestDB_ManualCompaction(t *testing.T) { func TestDB_BloomFilter(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ - BlockCache: opt.NoCache, - Filter: filter.NewBloomFilter(10), + DisableBlockCache: true, + Filter: filter.NewBloomFilter(10), }) defer h.close() @@ -2066,8 +2066,8 @@ func TestDB_GetProperties(t *testing.T) { func TestDB_GoleveldbIssue72and83(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ - WriteBuffer: 1 * opt.MiB, - CachedOpenFiles: 3, + WriteBuffer: 1 * opt.MiB, + OpenFilesCacheCapacity: 3, }) defer h.close() @@ -2200,7 +2200,7 @@ func TestDB_GoleveldbIssue72and83(t *testing.T) { func TestDB_TransientError(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ WriteBuffer: 128 * opt.KiB, - CachedOpenFiles: 3, + OpenFilesCacheCapacity: 3, DisableCompactionBackoff: true, }) defer h.close() @@ -2410,7 +2410,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) { CompactionTableSize: 43 * opt.KiB, CompactionExpandLimitFactor: 1, CompactionGPOverlapsFactor: 1, - BlockCache: opt.NoCache, + DisableBlockCache: true, } s, err := newSession(stor, o) if err != nil { diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go index aa7eb9a78..de10a85b8 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go @@ -17,14 +17,14 @@ import ( var _ = testutil.Defer(func() { Describe("Leveldb external", func() { o := &opt.Options{ - BlockCache: opt.NoCache, - BlockRestartInterval: 5, - BlockSize: 80, - Compression: opt.NoCompression, - CachedOpenFiles: -1, - Strict: opt.StrictAll, - WriteBuffer: 1000, - CompactionTableSize: 2000, + DisableBlockCache: true, + BlockRestartInterval: 5, + BlockSize: 80, + Compression: opt.NoCompression, + OpenFilesCacheCapacity: -1, + Strict: opt.StrictAll, + WriteBuffer: 1000, + CompactionTableSize: 2000, } Describe("write test", func() { diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go index 7f1c069dc..df7e1b441 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -20,8 +20,9 @@ const ( GiB = MiB * 1024 ) -const ( - DefaultBlockCacheSize = 8 * MiB +var ( + DefaultBlockCacher = LRUCacher + DefaultBlockCacheCapacity = 8 * MiB DefaultBlockRestartInterval = 16 DefaultBlockSize = 4 * KiB DefaultCompactionExpandLimitFactor = 25 @@ -33,7 +34,8 @@ const ( DefaultCompactionTotalSize = 10 * MiB DefaultCompactionTotalSizeMultiplier = 10.0 DefaultCompressionType = SnappyCompression - DefaultCachedOpenFiles = 500 + DefaultOpenFilesCacher = LRUCacher + DefaultOpenFilesCacheCapacity = 500 DefaultMaxMemCompationLevel = 2 DefaultNumLevel = 7 DefaultWriteBuffer = 4 * MiB @@ -41,22 +43,33 @@ const ( DefaultWriteL0SlowdownTrigger = 8 ) -type noCache struct{} +// Cacher is a caching algorithm. +type Cacher interface { + New(capacity int) cache.Cacher +} -func (noCache) SetCapacity(capacity int) {} -func (noCache) Capacity() int { return 0 } -func (noCache) Used() int { return 0 } -func (noCache) Size() int { return 0 } -func (noCache) NumObjects() int { return 0 } -func (noCache) GetNamespace(id uint64) cache.Namespace { return nil } -func (noCache) PurgeNamespace(id uint64, fin cache.PurgeFin) {} -func (noCache) ZapNamespace(id uint64) {} -func (noCache) Purge(fin cache.PurgeFin) {} -func (noCache) Zap() {} +type CacherFunc struct { + NewFunc func(capacity int) cache.Cacher +} -var NoCache cache.Cache = noCache{} +func (f *CacherFunc) New(capacity int) cache.Cacher { + if f.NewFunc != nil { + return f.NewFunc(capacity) + } + return nil +} -// Compression is the per-block compression algorithm to use. +func noCacher(int) cache.Cacher { return nil } + +var ( + // LRUCacher is the LRU-cache algorithm. + LRUCacher = &CacherFunc{cache.NewLRU} + + // NoCacher is the value to disable caching algorithm. + NoCacher = &CacherFunc{} +) + +// Compression is the 'sorted table' block compression algorithm to use. type Compression uint func (c Compression) String() string { @@ -133,16 +146,17 @@ type Options struct { // The default value is nil AltFilters []filter.Filter - // BlockCache provides per-block caching for LevelDB. Specify NoCache to - // disable block caching. + // BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching. + // Specify NoCacher to disable caching algorithm. // - // By default LevelDB will create LRU-cache with capacity of BlockCacheSize. - BlockCache cache.Cache + // The default value is LRUCacher. + BlockCacher Cacher - // BlockCacheSize defines the capacity of the default 'block cache'. + // BlockCacheCapacity defines the capacity of the 'sorted table' block caching. + // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher. // // The default value is 8MiB. - BlockCacheSize int + BlockCacheCapacity int // BlockRestartInterval is the number of keys between restart points for // delta encoding of keys. @@ -156,13 +170,6 @@ type Options struct { // The default value is 4KiB. BlockSize int - // CachedOpenFiles defines number of open files to kept around when not - // in-use, the counting includes still in-use files. - // Set this to negative value to disable caching. - // - // The default value is 500. - CachedOpenFiles int - // CompactionExpandLimitFactor limits compaction size after expanded. // This will be multiplied by table size limit at compaction target level. // @@ -237,11 +244,17 @@ type Options struct { // The default value uses the same ordering as bytes.Compare. Comparer comparer.Comparer - // Compression defines the per-block compression to use. + // Compression defines the 'sorted table' block compression to use. // // The default value (DefaultCompression) uses snappy compression. Compression Compression + // DisableBlockCache allows disable use of cache.Cache functionality on + // 'sorted table' block. + // + // The default value is false. + DisableBlockCache bool + // DisableCompactionBackoff allows disable compaction retry backoff. // // The default value is false. @@ -288,6 +301,18 @@ type Options struct { // The default is 7. NumLevel int + // OpenFilesCacher provides cache algorithm for open files caching. + // Specify NoCacher to disable caching algorithm. + // + // The default value is LRUCacher. + OpenFilesCacher Cacher + + // OpenFilesCacheCapacity defines the capacity of the open files caching. + // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher. + // + // The default value is 500. + OpenFilesCacheCapacity int + // Strict defines the DB strict level. Strict Strict @@ -320,18 +345,22 @@ func (o *Options) GetAltFilters() []filter.Filter { return o.AltFilters } -func (o *Options) GetBlockCache() cache.Cache { - if o == nil { +func (o *Options) GetBlockCacher() Cacher { + if o == nil || o.BlockCacher == nil { + return DefaultBlockCacher + } else if o.BlockCacher == NoCacher { return nil } - return o.BlockCache + return o.BlockCacher } -func (o *Options) GetBlockCacheSize() int { - if o == nil || o.BlockCacheSize <= 0 { - return DefaultBlockCacheSize +func (o *Options) GetBlockCacheCapacity() int { + if o == nil || o.BlockCacheCapacity == 0 { + return DefaultBlockCacheCapacity + } else if o.BlockCacheCapacity < 0 { + return 0 } - return o.BlockCacheSize + return o.BlockCacheCapacity } func (o *Options) GetBlockRestartInterval() int { @@ -348,15 +377,6 @@ func (o *Options) GetBlockSize() int { return o.BlockSize } -func (o *Options) GetCachedOpenFiles() int { - if o == nil || o.CachedOpenFiles == 0 { - return DefaultCachedOpenFiles - } else if o.CachedOpenFiles < 0 { - return 0 - } - return o.CachedOpenFiles -} - func (o *Options) GetCompactionExpandLimit(level int) int { factor := DefaultCompactionExpandLimitFactor if o != nil && o.CompactionExpandLimitFactor > 0 { @@ -477,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int { if o != nil { if o.MaxMemCompationLevel > 0 { level = o.MaxMemCompationLevel - } else if o.MaxMemCompationLevel == -1 { + } else if o.MaxMemCompationLevel < 0 { level = 0 } } @@ -494,6 +514,25 @@ func (o *Options) GetNumLevel() int { return o.NumLevel } +func (o *Options) GetOpenFilesCacher() Cacher { + if o == nil || o.OpenFilesCacher == nil { + return DefaultOpenFilesCacher + } + if o.OpenFilesCacher == NoCacher { + return nil + } + return o.OpenFilesCacher +} + +func (o *Options) GetOpenFilesCacheCapacity() int { + if o == nil || o.OpenFilesCacheCapacity == 0 { + return DefaultOpenFilesCacheCapacity + } else if o.OpenFilesCacheCapacity < 0 { + return 0 + } + return o.OpenFilesCacheCapacity +} + func (o *Options) GetStrict(strict Strict) bool { if o == nil || o.Strict == 0 { return DefaultStrict&strict != 0 diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go index 9c3538541..5def7ffe5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go @@ -7,7 +7,6 @@ package leveldb import ( - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" ) @@ -32,13 +31,6 @@ func (s *session) setOptions(o *opt.Options) { no.AltFilters[i] = &iFilter{filter} } } - // Block cache. - switch o.GetBlockCache() { - case nil: - no.BlockCache = cache.NewLRUCache(o.GetBlockCacheSize()) - case opt.NoCache: - no.BlockCache = nil - } // Comparer. s.icmp = &iComparer{o.GetComparer()} no.Comparer = s.icmp diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go index e0c908370..7e0a520a8 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go @@ -73,7 +73,7 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { stCompPtrs: make([]iKey, o.GetNumLevel()), } s.setOptions(o) - s.tops = newTableOps(s, s.o.GetCachedOpenFiles()) + s.tops = newTableOps(s) s.setVersion(newVersion(s)) s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") return @@ -82,9 +82,6 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { // Close session. func (s *session) close() { s.tops.close() - if bc := s.o.GetBlockCache(); bc != nil { - bc.Purge(nil) - } if s.manifest != nil { s.manifest.Close() } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go index 8584ee5da..202c921f5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go @@ -14,10 +14,8 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage" ) -// Logging. - type dropper struct { - s *session + s *session // Logging. file storage.File } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go index 125d17104..1b05ad92f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go @@ -286,10 +286,10 @@ func (x *tFilesSortByNum) Less(i, j int) bool { // Table operations. type tOps struct { - s *session - cache cache.Cache - cacheNS cache.Namespace - bpool *util.BufferPool + s *session + cache *cache.Cache + bcache *cache.Cache + bpool *util.BufferPool } // Creates an empty table and returns table writer. @@ -338,26 +338,28 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) { // Opens table. It returns a cache handle, which should // be released after use. -func (t *tOps) open(f *tFile) (ch cache.Handle, err error) { +func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) { num := f.file.Num() - ch = t.cacheNS.Get(num, func() (charge int, value interface{}) { + ch = t.cache.Get(0, num, func() (size int, value cache.Value) { var r storage.Reader r, err = f.file.Open() if err != nil { return 0, nil } - var bcacheNS cache.Namespace - if bc := t.s.o.GetBlockCache(); bc != nil { - bcacheNS = bc.GetNamespace(num) + var bcache *cache.CacheGetter + if t.bcache != nil { + bcache = &cache.CacheGetter{Cache: t.bcache, NS: num} } + var tr *table.Reader - tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcacheNS, t.bpool, t.s.o.Options) + tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcache, t.bpool, t.s.o.Options) if err != nil { r.Close() return 0, nil } return 1, tr + }) if ch == nil && err == nil { err = ErrClosed @@ -412,16 +414,14 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite // no one use the the table. func (t *tOps) remove(f *tFile) { num := f.file.Num() - t.cacheNS.Delete(num, func(exist, pending bool) { - if !pending { - if err := f.file.Remove(); err != nil { - t.s.logf("table@remove removing @%d %q", num, err) - } else { - t.s.logf("table@remove removed @%d", num) - } - if bc := t.s.o.GetBlockCache(); bc != nil { - bc.ZapNamespace(num) - } + t.cache.Delete(0, num, func() { + if err := f.file.Remove(); err != nil { + t.s.logf("table@remove removing @%d %q", num, err) + } else { + t.s.logf("table@remove removed @%d", num) + } + if t.bcache != nil { + t.bcache.EvictNS(num) } }) } @@ -429,18 +429,34 @@ func (t *tOps) remove(f *tFile) { // Closes the table ops instance. It will close all tables, // regadless still used or not. func (t *tOps) close() { - t.cache.Zap() t.bpool.Close() + t.cache.Close() + if t.bcache != nil { + t.bcache.Close() + } } // Creates new initialized table ops instance. -func newTableOps(s *session, cacheCap int) *tOps { - c := cache.NewLRUCache(cacheCap) +func newTableOps(s *session) *tOps { + var ( + cacher cache.Cacher + bcache *cache.Cache + ) + if s.o.GetOpenFilesCacheCapacity() > 0 { + cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity()) + } + if !s.o.DisableBlockCache { + var bcacher cache.Cacher + if s.o.GetBlockCacheCapacity() > 0 { + bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) + } + bcache = cache.NewCache(bcacher) + } return &tOps{ - s: s, - cache: c, - cacheNS: c.GetNamespace(0), - bpool: util.NewBufferPool(s.o.GetBlockSize() + 5), + s: s, + cache: cache.NewCache(cacher), + bcache: bcache, + bpool: util.NewBufferPool(s.o.GetBlockSize() + 5), } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go index 480a014ff..ef269fcc3 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go @@ -14,8 +14,6 @@ import ( "strings" "sync" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors" @@ -24,6 +22,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy" ) var ( @@ -509,7 +508,7 @@ type Reader struct { mu sync.RWMutex fi *storage.FileInfo reader io.ReaderAt - cache cache.Namespace + cache *cache.CacheGetter err error bpool *util.BufferPool // Options @@ -613,18 +612,22 @@ func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) { if r.cache != nil { - var err error - ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) { - if !fillCache { - return 0, nil - } - var b *block - b, err = r.readBlock(bh, verifyChecksum) - if err != nil { - return 0, nil - } - return cap(b.data), b - }) + var ( + err error + ch *cache.Handle + ) + if fillCache { + ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { + var b *block + b, err = r.readBlock(bh, verifyChecksum) + if err != nil { + return 0, nil + } + return cap(b.data), b + }) + } else { + ch = r.cache.Get(bh.offset, nil) + } if ch != nil { b, ok := ch.Value().(*block) if !ok { @@ -667,18 +670,22 @@ func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) { func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) { if r.cache != nil { - var err error - ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) { - if !fillCache { - return 0, nil - } - var b *filterBlock - b, err = r.readFilterBlock(bh) - if err != nil { - return 0, nil - } - return cap(b.data), b - }) + var ( + err error + ch *cache.Handle + ) + if fillCache { + ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { + var b *filterBlock + b, err = r.readFilterBlock(bh) + if err != nil { + return 0, nil + } + return cap(b.data), b + }) + } else { + ch = r.cache.Get(bh.offset, nil) + } if ch != nil { b, ok := ch.Value().(*filterBlock) if !ok { @@ -980,7 +987,7 @@ func (r *Reader) Release() { // The fi, cache and bpool is optional and can be nil. // // The returned table reader instance is goroutine-safe. -func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { +func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache *cache.CacheGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { if f == nil { return nil, errors.New("leveldb/table: nil file") } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go index 7a819d593..ccf6c9bde 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go @@ -12,12 +12,11 @@ import ( "fmt" "io" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy" ) func sharedPrefixLen(a, b []byte) int { diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go index d93c1b9db..552a17bfb 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go @@ -7,10 +7,15 @@ package snappy import ( "encoding/binary" "errors" + "io" ) -// ErrCorrupt reports that the input is invalid. -var ErrCorrupt = errors.New("snappy: corrupt input") +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") +) // DecodedLen returns the length of the decoded block. func DecodedLen(src []byte) (int, error) { @@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) { } return dst[:d], nil } + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxUncompressedChunkLen), + buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize), + } +} + +// Reader is an io.Reader than can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4]) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if !r.readFull(r.decoded[:n]) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)]) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + + } else { + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen]) { + return 0, r.err + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go index b2371db11..dda372422 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go @@ -6,6 +6,7 @@ package snappy import ( "encoding/binary" + "io" ) // We limit how far copy back-references can go, the same as the C++ code. @@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int { // This last factor dominates the blowup, so the final estimate is: return 32 + srcLen + srcLen/6 } + +// NewWriter returns a new Writer that compresses to w, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)), + } +} + +// Writer is an io.Writer than can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + enc []byte + buf [checksumSize + chunkHeaderSize]byte + wroteHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + w.wroteHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (n int, errRet error) { + if w.err != nil { + return 0, w.err + } + if !w.wroteHeader { + copy(w.enc, magicChunk) + if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil { + w.err = err + return n, err + } + w.wroteHeader = true + } + for len(p) > 0 { + var uncompressed []byte + if len(p) > maxUncompressedChunkLen { + uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + chunkType := uint8(chunkTypeCompressedData) + chunkBody, err := Encode(w.enc, uncompressed) + if err != nil { + w.err = err + return n, err + } + if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 { + chunkType, chunkBody = chunkTypeUncompressedData, uncompressed + } + + chunkLen := 4 + len(chunkBody) + w.buf[0] = chunkType + w.buf[1] = uint8(chunkLen >> 0) + w.buf[2] = uint8(chunkLen >> 8) + w.buf[3] = uint8(chunkLen >> 16) + w.buf[4] = uint8(checksum >> 0) + w.buf[5] = uint8(checksum >> 8) + w.buf[6] = uint8(checksum >> 16) + w.buf[7] = uint8(checksum >> 24) + if _, err = w.w.Write(w.buf[:]); err != nil { + w.err = err + return n, err + } + if _, err = w.w.Write(chunkBody); err != nil { + w.err = err + return n, err + } + n += len(uncompressed) + } + return n, nil +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go index 2f1b790d0..043bf3d81 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go @@ -8,6 +8,10 @@ // The C++ snappy implementation is at http://code.google.com/p/snappy/ package snappy +import ( + "hash/crc32" +) + /* Each encoded block begins with the varint-encoded length of the decoded data, followed by a sequence of chunks. Chunks begin and end on byte boundaries. The @@ -36,3 +40,29 @@ const ( tagCopy2 = 0x02 tagCopy4 = 0x03 ) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 bytes". + maxUncompressedChunkLen = 65536 +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go index 7ba839244..0623385b7 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go @@ -18,7 +18,10 @@ import ( "testing" ) -var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") +var ( + download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") + testdata = flag.String("testdata", "testdata", "Directory containing the test data") +) func roundtrip(b, ebuf, dbuf []byte) error { e, err := Encode(ebuf, b) @@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) { } func TestSmallRand(t *testing.T) { - rand.Seed(27354294) + rng := rand.New(rand.NewSource(27354294)) for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { - b[i] = uint8(rand.Uint32()) + for i := range b { + b[i] = uint8(rng.Uint32()) } if err := roundtrip(b, nil, nil); err != nil { t.Fatal(err) @@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) { func TestSmallRegular(t *testing.T) { for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { + for i := range b { b[i] = uint8(i%10 + 'a') } if err := roundtrip(b, nil, nil); err != nil { @@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) { } } +func cmp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("got %d bytes, want %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i]) + } + } + return nil +} + +func TestFramingFormat(t *testing.T) { + // src is comprised of alternating 1e5-sized sequences of random + // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen + // because it is larger than maxUncompressedChunkLen (64k). + src := make([]byte, 1e6) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 10; i++ { + if i%2 == 0 { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(rng.Intn(256)) + } + } else { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(i) + } + } + } + + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(src); err != nil { + t.Fatalf("Write: encoding: %v", err) + } + dst, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Fatalf("ReadAll: decoding: %v", err) + } + if err := cmp(dst, src); err != nil { + t.Fatal(err) + } +} + +func TestReaderReset(t *testing.T) { + gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000) + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(gold); err != nil { + t.Fatalf("Write: %v", err) + } + encoded, invalid, partial := buf.String(), "invalid", "partial" + r := NewReader(nil) + for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} { + if s == partial { + r.Reset(strings.NewReader(encoded)) + if _, err := r.Read(make([]byte, 101)); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + continue + } + r.Reset(strings.NewReader(s)) + got, err := ioutil.ReadAll(r) + switch s { + case encoded: + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + if err := cmp(got, gold); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + case invalid: + if err == nil { + t.Errorf("#%d: got nil error, want non-nil", i) + continue + } + } + } +} + +func TestWriterReset(t *testing.T) { + gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000) + var gots, wants [][]byte + const n = 20 + w, failed := NewWriter(nil), false + for i := 0; i <= n; i++ { + buf := new(bytes.Buffer) + w.Reset(buf) + want := gold[:len(gold)*i/n] + if _, err := w.Write(want); err != nil { + t.Errorf("#%d: Write: %v", i, err) + failed = true + continue + } + got, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Errorf("#%d: ReadAll: %v", i, err) + failed = true + continue + } + gots = append(gots, got) + wants = append(wants, want) + } + if failed { + return + } + for i := range gots { + if err := cmp(gots[i], wants[i]); err != nil { + t.Errorf("#%d: %v", i, err) + } + } +} + func benchDecode(b *testing.B, src []byte) { encoded, err := Encode(nil, src) if err != nil { @@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) { } } -func readFile(b *testing.B, filename string) []byte { +func readFile(b testing.TB, filename string) []byte { src, err := ioutil.ReadFile(filename) if err != nil { b.Fatalf("failed reading %s: %s", filename, err) @@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } // testFiles' values are copied directly from -// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc. +// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc // The label field is unused in snappy-go. var testFiles = []struct { label string @@ -152,29 +269,36 @@ var testFiles = []struct { }{ {"html", "html"}, {"urls", "urls.10K"}, - {"jpg", "house.jpg"}, - {"pdf", "mapreduce-osdi-1.pdf"}, + {"jpg", "fireworks.jpeg"}, + {"jpg_200", "fireworks.jpeg"}, + {"pdf", "paper-100k.pdf"}, {"html4", "html_x_4"}, - {"cp", "cp.html"}, - {"c", "fields.c"}, - {"lsp", "grammar.lsp"}, - {"xls", "kennedy.xls"}, {"txt1", "alice29.txt"}, {"txt2", "asyoulik.txt"}, {"txt3", "lcet10.txt"}, {"txt4", "plrabn12.txt"}, - {"bin", "ptt5"}, - {"sum", "sum"}, - {"man", "xargs.1"}, {"pb", "geo.protodata"}, {"gaviota", "kppkn.gtb"}, } // The test data files are present at this canonical URL. -const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/" +const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/" func downloadTestdata(basename string) (errRet error) { - filename := filepath.Join("testdata", basename) + filename := filepath.Join(*testdata, basename) + if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 { + return nil + } + + if !*download { + return fmt.Errorf("test data not found; skipping benchmark without the -download flag") + } + // Download the official snappy C++ implementation reference test data + // files for benchmarking. + if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create testdata: %s", err) + } + f, err := os.Create(filename) if err != nil { return fmt.Errorf("failed to create %s: %s", filename, err) @@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) { os.Remove(filename) } }() - resp, err := http.Get(baseURL + basename) + url := baseURL + basename + resp, err := http.Get(url) if err != nil { - return fmt.Errorf("failed to download %s: %s", baseURL+basename, err) + return fmt.Errorf("failed to download %s: %s", url, err) } defer resp.Body.Close() + if s := resp.StatusCode; s != http.StatusOK { + return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s)) + } _, err = io.Copy(f, resp.Body) if err != nil { - return fmt.Errorf("failed to write %s: %s", filename, err) + return fmt.Errorf("failed to download %s to %s: %s", url, filename, err) } return nil } func benchFile(b *testing.B, n int, decode bool) { - filename := filepath.Join("testdata", testFiles[n].filename) - if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 { - if !*download { - b.Fatal("test data not found; skipping benchmark without the -download flag") - } - // Download the official snappy C++ implementation reference test data - // files for benchmarking. - if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) { - b.Fatalf("failed to create testdata: %s", err) - } - for _, tf := range testFiles { - if err := downloadTestdata(tf.filename); err != nil { - b.Fatalf("failed to download testdata: %s", err) - } - } + if err := downloadTestdata(testFiles[n].filename); err != nil { + b.Fatalf("failed to download testdata: %s", err) } - data := readFile(b, filename) + data := readFile(b, filepath.Join(*testdata, testFiles[n].filename)) if decode { benchDecode(b, data) } else { @@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } -func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) } -func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) } -func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) } -func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) } -func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) } -func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) } func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } @@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } -func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) } -func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) } -func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) } -func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) } -func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) } -func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }