Merge pull request #958 from jbenet/godeps

godeps update
This commit is contained in:
Juan Batiz-Benet 2015-03-24 15:39:03 -07:00
commit b4da412b8f
89 changed files with 3437 additions and 5343 deletions

32
Godeps/Godeps.json generated
View File

@ -7,7 +7,7 @@
"Deps": [
{
"ImportPath": "bazil.org/fuse",
"Rev": "489e985dcb429718328be2bb055adb685c4ae1e3"
"Rev": "79d103f5608724e3ccee0a10daccc5e4aff03591"
},
{
"ImportPath": "code.google.com/p/go-uuid/uuid",
@ -33,8 +33,8 @@
},
{
"ImportPath": "github.com/Sirupsen/logrus",
"Comment": "v0.6.5",
"Rev": "c0f7e35ed2e48f188c37581b4b743cf7383f85c6"
"Comment": "v0.7.1",
"Rev": "3fc34d061b9c78a70db853c7cb6b0576b6d4f32d"
},
{
"ImportPath": "github.com/braintree/manners",
@ -67,7 +67,7 @@
},
{
"ImportPath": "github.com/coreos/go-semver/semver",
"Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa"
"Rev": "568e959cd89871e61434c1143528d9162da89ef2"
},
{
"ImportPath": "github.com/crowdmob/goamz/aws",
@ -102,14 +102,6 @@
"Comment": "v0.5.1",
"Rev": "27a863cdffdb0998d13e1e11992b18489aeeaa25"
},
{
"ImportPath": "github.com/gorilla/context",
"Rev": "14f550f51af52180c2eefed15e5fd18d63c0a64a"
},
{
"ImportPath": "github.com/gorilla/mux",
"Rev": "4b8fbc56f3b2400a7c7ea3dba9b3539787c486b6"
},
{
"ImportPath": "github.com/h2so5/utp",
"Rev": "654d875bb65e96729678180215cf080fe2810371"
@ -141,7 +133,7 @@
},
{
"ImportPath": "github.com/jbenet/go-base58",
"Rev": "568a28d73fd97651d3442392036a658b6976eed5"
"Rev": "6237cf65f3a6f7111cd8a42be3590df99a66bc7d"
},
{
"ImportPath": "github.com/jbenet/go-ctxgroup",
@ -195,7 +187,7 @@
},
{
"ImportPath": "github.com/jbenet/go-reuseport",
"Rev": "096958438ae3683c9f3c8ae0f7139b5ce600e6a8"
"Rev": "c71a70ef82a7acb87ad77704de1be7b9cd776a55"
},
{
"ImportPath": "github.com/jbenet/go-sockaddr/net",
@ -211,7 +203,7 @@
},
{
"ImportPath": "github.com/kardianos/osext",
"Rev": "ccfcd0245381f0c94c68f50626665eed3c6b726a"
"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
},
{
"ImportPath": "github.com/kr/binarydist",
@ -223,23 +215,23 @@
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "871eee0a7546bb7d1b2795142e29c4534abc49b3"
"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
},
{
"ImportPath": "github.com/syndtr/gosnappy/snappy",
"Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
},
{
"ImportPath": "golang.org/x/crypto/blowfish",
"Rev": "1351f936d976c60a0a48d728281922cf63eafb8d"
"Rev": "b7d6bf2c61544745a02f83dec90393985fc3a065"
},
{
"ImportPath": "golang.org/x/crypto/sha3",
"Rev": "1351f936d976c60a0a48d728281922cf63eafb8d"
"Rev": "b7d6bf2c61544745a02f83dec90393985fc3a065"
},
{
"ImportPath": "golang.org/x/net/context",
"Rev": "b6fdb7d8a4ccefede406f8fe0f017fb58265054c"
"Rev": "7dbad50ab5b31073856416cdcfeb2796d682f844"
},
{
"ImportPath": "gopkg.in/fsnotify.v1",

View File

@ -4,33 +4,14 @@ import (
"syscall"
)
type getxattrError struct {
error
}
func (getxattrError) Errno() Errno {
return Errno(syscall.ENOATTR)
}
// getxattr return value for "extended attribute does not exist" is
// ENOATTR on OS X, and ENODATA on Linux and apparently at least
// NetBSD. There may be a #define ENOATTR too, but the value is
// ENODATA in the actual syscalls. ENOATTR is not in any of the
// standards, ENODATA exists but is only used for STREAMs.
//
// https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/getxattr.2.html
// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013090.html
// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013097.html
// http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html
func translateGetxattrError(err error) error {
ferr, ok := err.(ErrorNumber)
if !ok {
return err
}
if ferr.Errno() != ENODATA {
return err
}
return getxattrError{err}
const (
ENOATTR = Errno(syscall.ENOATTR)
)
const (
errNoXattr = ENOATTR
)
func init() {
errnoNames[errNoXattr] = "ENOATTR"
}

15
Godeps/_workspace/src/bazil.org/fuse/error_freebsd.go generated vendored Normal file
View File

@ -0,0 +1,15 @@
package fuse
import "syscall"
const (
ENOATTR = Errno(syscall.ENOATTR)
)
const (
errNoXattr = ENOATTR
)
func init() {
errnoNames[errNoXattr] = "ENOATTR"
}

17
Godeps/_workspace/src/bazil.org/fuse/error_linux.go generated vendored Normal file
View File

@ -0,0 +1,17 @@
package fuse
import (
"syscall"
)
const (
ENODATA = Errno(syscall.ENODATA)
)
const (
errNoXattr = ENODATA
)
func init() {
errnoNames[errNoXattr] = "ENODATA"
}

View File

@ -1,7 +1,31 @@
// +build !darwin
package fuse
func translateGetxattrError(err error) error {
return err
}
// There is very little commonality in extended attribute errors
// across platforms.
//
// getxattr return value for "extended attribute does not exist" is
// ENOATTR on OS X, and ENODATA on Linux and apparently at least
// NetBSD. There may be a #define ENOATTR on Linux too, but the value
// is ENODATA in the actual syscalls. FreeBSD and OpenBSD have no
// ENODATA, only ENOATTR. ENOATTR is not in any of the standards,
// ENODATA exists but is only used for STREAMs.
//
// Each platform will define it a errNoXattr constant, and this file
// will enforce that it implements the right interfaces and hide the
// implementation.
//
// https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/getxattr.2.html
// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013090.html
// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013097.html
// http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html
// http://www.freebsd.org/cgi/man.cgi?query=extattr_get_file&sektion=2
// http://nixdoc.net/man-pages/openbsd/man2/extattr_get_file.2.html
// ErrNoXattr is a platform-independent error value meaning the
// extended attribute was not found. It can be used to respond to
// GetxattrRequest and such.
const ErrNoXattr = errNoXattr
var _ error = ErrNoXattr
var _ Errno = ErrNoXattr
var _ ErrorNumber = ErrNoXattr

View File

@ -12,3 +12,15 @@ type MountInfo struct {
func GetMountInfo(mnt string) (*MountInfo, error) {
return getMountInfo(mnt)
}
// cstr converts a nil-terminated C string into a Go string
func cstr(ca []int8) string {
s := make([]byte, 0, len(ca))
for _, c := range ca {
if c == 0x00 {
break
}
s = append(s, byte(c))
}
return string(s)
}

View File

@ -5,18 +5,6 @@ import (
"syscall"
)
// cstr converts a nil-terminated C string into a Go string
func cstr(ca []int8) string {
s := make([]byte, 0, len(ca))
for _, c := range ca {
if c == 0x00 {
break
}
s = append(s, byte(c))
}
return string(s)
}
var re = regexp.MustCompile(`\\(.)`)
// unescape removes backslash-escaping. The escaped characters are not

View File

@ -0,0 +1,7 @@
package fstestutil
import "errors"
func getMountInfo(mnt string) (*MountInfo, error) {
return nil, errors.New("FreeBSD has no useful mount information")
}

View File

@ -20,8 +20,7 @@ func (w *Writes) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.W
n, err := w.buf.Write(req.Data)
resp.Size = n
if err != nil {
// TODO hiding error
return fuse.EIO
return err
}
return nil
}
@ -293,7 +292,7 @@ var _ = fs.NodeGetxattrer(&Getxattrs{})
func (r *Getxattrs) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
tmp := *req
r.rec.RecordRequest(&tmp)
return fuse.ENODATA
return fuse.ErrNoXattr
}
// RecordedGetxattr returns information about the Getxattr request.
@ -318,7 +317,7 @@ var _ = fs.NodeListxattrer(&Listxattrs{})
func (r *Listxattrs) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
tmp := *req
r.rec.RecordRequest(&tmp)
return fuse.ENODATA
return fuse.ErrNoXattr
}
// RecordedListxattr returns information about the Listxattr request.

View File

@ -5,6 +5,7 @@ import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
)
// SimpleFS is a trivial FS that just implements the Root method.
@ -27,3 +28,21 @@ func (f File) Attr() fuse.Attr { return fuse.Attr{Mode: 0666} }
type Dir struct{}
func (f Dir) Attr() fuse.Attr { return fuse.Attr{Mode: os.ModeDir | 0777} }
// ChildMap is a directory with child nodes looked up from a map.
type ChildMap map[string]fs.Node
var _ = fs.Node(ChildMap{})
var _ = fs.NodeStringLookuper(ChildMap{})
func (f ChildMap) Attr() fuse.Attr {
return fuse.Attr{Mode: os.ModeDir | 0777}
}
func (f ChildMap) Lookup(ctx context.Context, name string) (fs.Node, error) {
child, ok := f[name]
if !ok {
return nil, fuse.ENOENT
}
return child, nil
}

View File

@ -0,0 +1,67 @@
package fs_test
import (
"errors"
"flag"
"os"
"os/exec"
"path/filepath"
"testing"
)
var childHelpers = map[string]func(){}
type childProcess struct {
name string
fn func()
}
var _ flag.Value = (*childProcess)(nil)
func (c *childProcess) String() string {
return c.name
}
func (c *childProcess) Set(s string) error {
fn, ok := childHelpers[s]
if !ok {
return errors.New("helper not found")
}
c.name = s
c.fn = fn
return nil
}
var childMode childProcess
func init() {
flag.Var(&childMode, "fuse.internal.child", "internal use only")
}
// childCmd prepares a test function to be run in a subprocess, with
// childMode set to true. Caller must still call Run or Start.
//
// Re-using the test executable as the subprocess is useful because
// now test executables can e.g. be cross-compiled, transferred
// between hosts, and run in settings where the whole Go development
// environment is not installed.
func childCmd(childName string) (*exec.Cmd, error) {
// caller may set cwd, so we can't rely on relative paths
executable, err := filepath.Abs(os.Args[0])
if err != nil {
return nil, err
}
cmd := exec.Command(executable, "-fuse.internal.child="+childName)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd, nil
}
func TestMain(m *testing.M) {
flag.Parse()
if childMode.fn != nil {
childMode.fn()
os.Exit(0)
}
os.Exit(m.Run())
}

View File

@ -101,6 +101,15 @@ type NodeGetattrer interface {
type NodeSetattrer interface {
// Setattr sets the standard metadata for the receiver.
//
// Note, this is also used to communicate changes in the size of
// the file. Not implementing Setattr causes writes to be unable
// to grow the file (except with OpenDirectIO, which bypasses that
// mechanism).
//
// req.Valid is a bitmask of what fields are actually being set.
// For example, the method should not change the mode of the file
// unless req.Valid.Mode() is true.
Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error
}
@ -203,9 +212,7 @@ type NodeGetxattrer interface {
// Getxattr gets an extended attribute by the given name from the
// node.
//
// If there is no xattr by that name, returns fuse.ENODATA. This
// will be translated to the platform-specific correct error code
// by the framework.
// If there is no xattr by that name, returns fuse.ErrNoXattr.
Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error
}
@ -223,9 +230,7 @@ type NodeSetxattrer interface {
type NodeRemovexattrer interface {
// Removexattr removes an extended attribute for the name.
//
// If there is no xattr by that name, returns fuse.ENODATA. This
// will be translated to the platform-specific correct error code
// by the framework.
// If there is no xattr by that name, returns fuse.ErrNoXattr.
Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error
}
@ -279,10 +284,29 @@ type HandleReadDirAller interface {
}
type HandleReader interface {
// Read requests to read data from the handle.
//
// There is a page cache in the kernel that normally submits only
// page-aligned reads spanning one or more pages. However, you
// should not rely on this. To see individual requests as
// submitted by the file system clients, set OpenDirectIO.
//
// Note that reads beyond the size of the file as reported by Attr
// are not even attempted (except in OpenDirectIO mode).
Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error
}
type HandleWriter interface {
// Write requests to write data into the handle.
//
// There is a writeback page cache in the kernel that normally submits
// only page-aligned writes spanning one or more pages. However,
// you should not rely on this. To see individual requests as
// submitted by the file system clients, set OpenDirectIO.
//
// Note that file size changes are communicated through Setattr.
// Writes beyond the size of the file as reported by Attr are not
// even attempted (except in OpenDirectIO mode).
Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error
}

View File

@ -3,14 +3,11 @@ package fs_test
import (
"bytes"
"errors"
"flag"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"syscall"
@ -40,55 +37,6 @@ func init() {
fstestutil.DebugByDefault()
}
var childMode bool
func init() {
flag.BoolVar(&childMode, "fuse.internal.childmode", false, "internal use only")
}
// childCmd prepares a test function to be run in a subprocess, with
// childMode set to true. Caller must still call Run or Start.
//
// Re-using the test executable as the subprocess is useful because
// now test executables can e.g. be cross-compiled, transferred
// between hosts, and run in settings where the whole Go development
// environment is not installed.
func childCmd(testName string) (*exec.Cmd, error) {
// caller may set cwd, so we can't rely on relative paths
executable, err := filepath.Abs(os.Args[0])
if err != nil {
return nil, err
}
testName = regexp.QuoteMeta(testName)
cmd := exec.Command(executable, "-test.run=^"+testName+"$", "-fuse.internal.childmode")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd, nil
}
// childMapFS is an FS with one fixed child named "child".
type childMapFS map[string]fs.Node
var _ = fs.FS(childMapFS{})
var _ = fs.Node(childMapFS{})
var _ = fs.NodeStringLookuper(childMapFS{})
func (f childMapFS) Attr() fuse.Attr {
return fuse.Attr{Inode: 1, Mode: os.ModeDir | 0777}
}
func (f childMapFS) Root() (fs.Node, error) {
return f, nil
}
func (f childMapFS) Lookup(ctx context.Context, name string) (fs.Node, error) {
child, ok := f[name]
if !ok {
return nil, fuse.ENOENT
}
return child, nil
}
// symlink can be embedded in a struct to make it look like a symlink.
type symlink struct {
target string
@ -271,7 +219,7 @@ func testReadAll(t *testing.T, path string) {
func TestReadAll(t *testing.T) {
t.Parallel()
mnt, err := fstestutil.MountedT(t, childMapFS{"child": readAll{}})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": readAll{}}})
if err != nil {
t.Fatal(err)
}
@ -300,7 +248,7 @@ func (readWithHandleRead) Read(ctx context.Context, req *fuse.ReadRequest, resp
func TestReadAllWithHandleRead(t *testing.T) {
t.Parallel()
mnt, err := fstestutil.MountedT(t, childMapFS{"child": readWithHandleRead{}})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": readWithHandleRead{}}})
if err != nil {
t.Fatal(err)
}
@ -319,7 +267,7 @@ type release struct {
func TestRelease(t *testing.T) {
t.Parallel()
r := &release{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": r})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": r}})
if err != nil {
t.Fatal(err)
}
@ -346,7 +294,7 @@ type write struct {
func TestWrite(t *testing.T) {
t.Parallel()
w := &write{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": w})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}})
if err != nil {
t.Fatal(err)
}
@ -393,7 +341,7 @@ type writeLarge struct {
func TestWriteLarge(t *testing.T) {
t.Parallel()
w := &write{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": w})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}})
if err != nil {
t.Fatal(err)
}
@ -440,7 +388,7 @@ type writeTruncateFlush struct {
func TestWriteTruncateFlush(t *testing.T) {
t.Parallel()
w := &writeTruncateFlush{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": w})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}})
if err != nil {
t.Fatal(err)
}
@ -838,7 +786,7 @@ func (dataHandleTest) Open(ctx context.Context, req *fuse.OpenRequest, resp *fus
func TestDataHandle(t *testing.T) {
t.Parallel()
f := &dataHandleTest{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -883,7 +831,7 @@ func TestInterrupt(t *testing.T) {
t.Parallel()
f := &interrupt{}
f.hanging = make(chan struct{}, 1)
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -947,7 +895,7 @@ type truncate struct {
func testTruncate(t *testing.T, toSize int64) {
t.Parallel()
f := &truncate{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -988,7 +936,7 @@ type ftruncate struct {
func testFtruncate(t *testing.T, toSize int64) {
t.Parallel()
f := &ftruncate{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1038,7 +986,7 @@ type truncateWithOpen struct {
func TestTruncateWithOpen(t *testing.T) {
t.Parallel()
f := &truncateWithOpen{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1165,7 +1113,7 @@ func (f *chmod) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
func TestChmod(t *testing.T) {
t.Parallel()
f := &chmod{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1199,7 +1147,7 @@ func (f *open) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
func TestOpen(t *testing.T) {
t.Parallel()
f := &open{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1307,7 +1255,7 @@ func (f *getxattr) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp
func TestGetxattr(t *testing.T) {
t.Parallel()
f := &getxattr{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1343,7 +1291,7 @@ func (f *getxattrTooSmall) Getxattr(ctx context.Context, req *fuse.GetxattrReque
func TestGetxattrTooSmall(t *testing.T) {
t.Parallel()
f := &getxattrTooSmall{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1374,7 +1322,7 @@ func (f *getxattrSize) Getxattr(ctx context.Context, req *fuse.GetxattrRequest,
func TestGetxattrSize(t *testing.T) {
t.Parallel()
f := &getxattrSize{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1406,7 +1354,7 @@ func (f *listxattr) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, r
func TestListxattr(t *testing.T) {
t.Parallel()
f := &listxattr{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1445,7 +1393,7 @@ func (f *listxattrTooSmall) Listxattr(ctx context.Context, req *fuse.ListxattrRe
func TestListxattrTooSmall(t *testing.T) {
t.Parallel()
f := &listxattrTooSmall{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1476,7 +1424,7 @@ func (f *listxattrSize) Listxattr(ctx context.Context, req *fuse.ListxattrReques
func TestListxattrSize(t *testing.T) {
t.Parallel()
f := &listxattrSize{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1507,7 +1455,7 @@ func testSetxattr(t *testing.T, size int) {
t.Parallel()
f := &setxattr{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1560,7 +1508,7 @@ type removexattr struct {
func TestRemovexattr(t *testing.T) {
t.Parallel()
f := &removexattr{}
mnt, err := fstestutil.MountedT(t, childMapFS{"child": f})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": f}})
if err != nil {
t.Fatal(err)
}
@ -1681,6 +1629,54 @@ func (f *inMemoryFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *
return nil
}
const mmapSize = 16 * 4096
var mmapWrites = map[int]byte{
10: 'a',
4096: 'b',
4097: 'c',
mmapSize - 4096: 'd',
mmapSize - 1: 'z',
}
func helperMmap() {
f, err := os.Create("child")
if err != nil {
log.Fatalf("Create: %v", err)
}
defer f.Close()
data, err := syscall.Mmap(int(f.Fd()), 0, mmapSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
log.Fatalf("Mmap: %v", err)
}
for i, b := range mmapWrites {
data[i] = b
}
if err := syscallx.Msync(data, syscall.MS_SYNC); err != nil {
log.Fatalf("Msync: %v", err)
}
if err := syscall.Munmap(data); err != nil {
log.Fatalf("Munmap: %v", err)
}
if err := f.Sync(); err != nil {
log.Fatalf("Fsync = %v", err)
}
err = f.Close()
if err != nil {
log.Fatalf("Close: %v", err)
}
}
func init() {
childHelpers["mmap"] = helperMmap
}
type mmap struct {
inMemoryFile
// We don't actually care about whether the fsync happened or not;
@ -1690,65 +1686,21 @@ type mmap struct {
}
func TestMmap(t *testing.T) {
const size = 16 * 4096
writes := map[int]byte{
10: 'a',
4096: 'b',
4097: 'c',
size - 4096: 'd',
size - 1: 'z',
w := &mmap{}
w.data = make([]byte, mmapSize)
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}})
if err != nil {
t.Fatal(err)
}
defer mnt.Close()
// Run the mmap-using parts of the test in a subprocess, to avoid
// an intentional page fault hanging the whole process (because it
// would need to be served by the same process, and there might
// not be a thread free to do that). Merely bumping GOMAXPROCS is
// not enough to prevent the hangs reliably.
if childMode {
f, err := os.Create("child")
if err != nil {
t.Fatalf("Create: %v", err)
}
defer f.Close()
data, err := syscall.Mmap(int(f.Fd()), 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
t.Fatalf("Mmap: %v", err)
}
for i, b := range writes {
data[i] = b
}
if err := syscallx.Msync(data, syscall.MS_SYNC); err != nil {
t.Fatalf("Msync: %v", err)
}
if err := syscall.Munmap(data); err != nil {
t.Fatalf("Munmap: %v", err)
}
if err := f.Sync(); err != nil {
t.Fatalf("Fsync = %v", err)
}
err = f.Close()
if err != nil {
t.Fatalf("Close: %v", err)
}
return
}
w := &mmap{}
w.data = make([]byte, size)
mnt, err := fstestutil.MountedT(t, childMapFS{"child": w})
if err != nil {
t.Fatal(err)
}
defer mnt.Close()
child, err := childCmd("TestMmap")
child, err := childCmd("mmap")
if err != nil {
t.Fatal(err)
}
@ -1758,12 +1710,12 @@ func TestMmap(t *testing.T) {
}
got := w.data
if g, e := len(got), size; g != e {
if g, e := len(got), mmapSize; g != e {
t.Fatalf("bad write length: %d != %d", g, e)
}
for i, g := range got {
// default '\x00' for writes[i] is good here
if e := writes[i]; g != e {
if e := mmapWrites[i]; g != e {
t.Errorf("wrong byte at offset %d: %q != %q", i, g, e)
}
}
@ -1790,7 +1742,7 @@ func (directRead) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.Re
func TestDirectRead(t *testing.T) {
t.Parallel()
mnt, err := fstestutil.MountedT(t, childMapFS{"child": directRead{}})
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": directRead{}}})
if err != nil {
t.Fatal(err)
}
@ -1798,3 +1750,50 @@ func TestDirectRead(t *testing.T) {
testReadAll(t, mnt.Dir+"/child")
}
// Test direct Write.
type directWrite struct {
fstestutil.File
record.Writes
}
// explicitly not defining Attr / Setattr and managing Size
func (f *directWrite) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
// do not allow the kernel to use page cache
resp.Flags |= fuse.OpenDirectIO
return f, nil
}
func TestDirectWrite(t *testing.T) {
t.Parallel()
w := &directWrite{}
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.ChildMap{"child": w}})
if err != nil {
t.Fatal(err)
}
defer mnt.Close()
f, err := os.OpenFile(mnt.Dir+"/child", os.O_RDWR, 0666)
if err != nil {
t.Fatalf("Create: %v", err)
}
defer f.Close()
n, err := f.Write([]byte(hi))
if err != nil {
t.Fatalf("Write: %v", err)
}
if n != len(hi) {
t.Fatalf("short write; n=%d; hi=%d", n, len(hi))
}
err = f.Close()
if err != nil {
t.Fatalf("Close: %v", err)
}
if got := string(w.RecordedWriteData()); got != hi {
t.Errorf("write = %q, want %q", got, hi)
}
}

View File

@ -47,11 +47,17 @@
//
// Op(ctx context.Context, req *OpRequest, resp *OpResponse) Error
//
// where Op is the name of a FUSE operation. Op reads request parameters
// from req and writes results to resp. An operation whose only result is
// the error result omits the resp parameter. Multiple goroutines may call
// service methods simultaneously; the methods being called are responsible
// for appropriate synchronization.
// where Op is the name of a FUSE operation. Op reads request
// parameters from req and writes results to resp. An operation whose
// only result is the error result omits the resp parameter.
//
// Multiple goroutines may call service methods simultaneously; the
// methods being called are responsible for appropriate
// synchronization.
//
// The operation must not hold on to the request or response,
// including any []byte fields such as WriteRequest.Data or
// SetxattrRequest.Xattr.
//
// Errors
//
@ -79,9 +85,12 @@
//
// Authentication
//
// All requests types embed a Header, meaning that the method can inspect
// req.Pid, req.Uid, and req.Gid as necessary to implement permission checking.
// Alternately, XXX.
// All requests types embed a Header, meaning that the method can
// inspect req.Pid, req.Uid, and req.Gid as necessary to implement
// permission checking. The kernel FUSE layer normally prevents other
// users from accessing the FUSE file system (to change this, see
// AllowOther, AllowRoot), but does not enforce access modes (to
// change this, see DefaultPermissions).
//
// Mount Options
//
@ -90,8 +99,6 @@
//
package fuse
// BUG(rsc): The mount code for FreeBSD has not been written yet.
import (
"bytes"
"encoding/json"
@ -241,7 +248,6 @@ const (
// See also fs.Intr.
EINTR = Errno(syscall.EINTR)
ENODATA = Errno(syscall.ENODATA)
ERANGE = Errno(syscall.ERANGE)
ENOTSUP = Errno(syscall.ENOTSUP)
EEXIST = Errno(syscall.EEXIST)
@ -252,14 +258,13 @@ const (
const DefaultErrno = EIO
var errnoNames = map[Errno]string{
ENOSYS: "ENOSYS",
ESTALE: "ESTALE",
ENOENT: "ENOENT",
EIO: "EIO",
EPERM: "EPERM",
EINTR: "EINTR",
ENODATA: "ENODATA",
EEXIST: "EEXIST",
ENOSYS: "ENOSYS",
ESTALE: "ESTALE",
ENOENT: "ENOENT",
EIO: "EIO",
EPERM: "EPERM",
EINTR: "EINTR",
EEXIST: "EEXIST",
}
// Errno implements Error and ErrorNumber using a syscall.Errno.
@ -1115,9 +1120,6 @@ func (a *Attr) attr() (out attr) {
out.Mode |= syscall.S_ISGID
}
out.Nlink = a.Nlink
if out.Nlink < 1 {
out.Nlink = 1
}
out.Uid = a.Uid
out.Gid = a.Gid
out.Rdev = a.Rdev
@ -1195,11 +1197,6 @@ func (r *GetxattrRequest) Respond(resp *GetxattrResponse) {
}
}
func (r *GetxattrRequest) RespondError(err error) {
err = translateGetxattrError(err)
r.Header.RespondError(err)
}
// A GetxattrResponse is the response to a GetxattrRequest.
type GetxattrResponse struct {
Xattr []byte
@ -1271,11 +1268,6 @@ func (r *RemovexattrRequest) Respond() {
r.respond(out, unsafe.Sizeof(*out))
}
func (r *RemovexattrRequest) RespondError(err error) {
err = translateGetxattrError(err)
r.Header.RespondError(err)
}
// A SetxattrRequest asks to set an extended attribute associated with a file.
type SetxattrRequest struct {
Header `json:"-"`
@ -1321,11 +1313,6 @@ func (r *SetxattrRequest) Respond() {
r.respond(out, unsafe.Sizeof(*out))
}
func (r *SetxattrRequest) RespondError(err error) {
err = translateGetxattrError(err)
r.Header.RespondError(err)
}
// A LookupRequest asks to look up the given name in the directory named by r.Node.
type LookupRequest struct {
Header `json:"-"`

View File

@ -0,0 +1,60 @@
package fuse
import "time"
type attr struct {
Ino uint64
Size uint64
Blocks uint64
Atime uint64
Mtime uint64
Ctime uint64
AtimeNsec uint32
MtimeNsec uint32
CtimeNsec uint32
Mode uint32
Nlink uint32
Uid uint32
Gid uint32
Rdev uint32
}
func (a *attr) Crtime() time.Time {
return time.Time{}
}
func (a *attr) SetCrtime(s uint64, ns uint32) {
// ignored on freebsd
}
func (a *attr) SetFlags(f uint32) {
// ignored on freebsd
}
type setattrIn struct {
setattrInCommon
}
func (in *setattrIn) BkupTime() time.Time {
return time.Time{}
}
func (in *setattrIn) Chgtime() time.Time {
return time.Time{}
}
func (in *setattrIn) Flags() uint32 {
return 0
}
func openFlags(flags uint32) OpenFlags {
return OpenFlags(flags)
}
type getxattrIn struct {
getxattrInCommon
}
type setxattrIn struct {
setxattrInCommon
}

View File

@ -59,7 +59,7 @@ func callMount(dir string, conf *MountConfig, f *os.File, ready chan<- struct{},
if strings.Contains(k, ",") || strings.Contains(v, ",") {
// Silly limitation but the mount helper does not
// understand any escaping. See TestMountOptionCommaError.
return fmt.Errorf("mount options cannot contain commas on OS X: %q=%q", k, v)
return fmt.Errorf("mount options cannot contain commas on darwin: %q=%q", k, v)
}
}
cmd := exec.Command(
@ -89,7 +89,7 @@ func callMount(dir string, conf *MountConfig, f *os.File, ready chan<- struct{},
return err
}
go func() {
err = cmd.Wait()
err := cmd.Wait()
if err != nil {
if buf.Len() > 0 {
output := buf.Bytes()

41
Godeps/_workspace/src/bazil.org/fuse/mount_freebsd.go generated vendored Normal file
View File

@ -0,0 +1,41 @@
package fuse
import (
"fmt"
"os"
"os/exec"
"strings"
)
func mount(dir string, conf *MountConfig, ready chan<- struct{}, errp *error) (*os.File, error) {
for k, v := range conf.options {
if strings.Contains(k, ",") || strings.Contains(v, ",") {
// Silly limitation but the mount helper does not
// understand any escaping. See TestMountOptionCommaError.
return nil, fmt.Errorf("mount options cannot contain commas on FreeBSD: %q=%q", k, v)
}
}
f, err := os.OpenFile("/dev/fuse", os.O_RDWR, 0000)
if err != nil {
*errp = err
return nil, err
}
cmd := exec.Command(
"/sbin/mount_fusefs",
"--safe",
"-o", conf.getOptions(),
"3",
dir,
)
cmd.ExtraFiles = []*os.File{f}
out, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("mount_fusefs: %q, %v", out, err)
}
close(ready)
return f, nil
}

View File

@ -5,6 +5,10 @@ import (
"strings"
)
func dummyOption(conf *MountConfig) error {
return nil
}
// MountConfig holds the configuration for a mount operation.
// Use it by passing MountOption values to Mount.
type MountConfig struct {
@ -37,6 +41,8 @@ type MountOption func(*MountConfig) error
// FSName sets the file system name (also called source) that is
// visible in the list of mounted file systems.
//
// FreeBSD ignores this option.
func FSName(name string) MountOption {
return func(conf *MountConfig) error {
conf.options["fsname"] = name
@ -49,6 +55,7 @@ func FSName(name string) MountOption {
// `fuse.foo`.
//
// OS X ignores this option.
// FreeBSD ignores this option.
func Subtype(fstype string) MountOption {
return func(conf *MountConfig) error {
conf.options["subtype"] = fstype
@ -89,6 +96,8 @@ func AllowOther() MountOption {
// AllowRoot allows other users to access the file system.
//
// Only one of AllowOther or AllowRoot can be used.
//
// FreeBSD ignores this option.
func AllowRoot() MountOption {
return func(conf *MountConfig) error {
if _, ok := conf.options["allow_other"]; ok {
@ -98,3 +107,26 @@ func AllowRoot() MountOption {
return nil
}
}
// DefaultPermissions makes the kernel enforce access control based on
// the file mode (as in chmod).
//
// Without this option, the Node itself decides what is and is not
// allowed. This is normally ok because FUSE file systems cannot be
// accessed by other users without AllowOther/AllowRoot.
//
// FreeBSD ignores this option.
func DefaultPermissions() MountOption {
return func(conf *MountConfig) error {
conf.options["default_permissions"] = ""
return nil
}
}
// ReadOnly makes the mount read-only.
func ReadOnly() MountOption {
return func(conf *MountConfig) error {
conf.options["ro"] = ""
return nil
}
}

View File

@ -1,27 +0,0 @@
package fuse_test
import (
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
)
func TestMountOptionCommaError(t *testing.T) {
t.Parallel()
// this test is not tied to FSName, but needs just some option
// with string content
var name = "FuseTest,Marker"
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}},
fuse.FSName(name),
)
switch {
case err == nil:
mnt.Close()
t.Fatal("expected an error about commas")
case err.Error() == `mount options cannot contain commas on OS X: "fsname"="FuseTest,Marker"`:
// all good
default:
t.Fatalf("expected an error about commas, got: %v", err)
}
}

View File

@ -0,0 +1,9 @@
package fuse
func localVolume(conf *MountConfig) error {
return nil
}
func volumeName(name string) MountOption {
return dummyOption
}

View File

@ -0,0 +1,6 @@
package fuse
// for TestMountOptionCommaError
func ForTestSetMountOption(conf *MountConfig, k, v string) {
conf.options[k] = v
}

View File

@ -1,9 +1,5 @@
package fuse
func dummyOption(conf *MountConfig) error {
return nil
}
func localVolume(conf *MountConfig) error {
return nil
}

View File

@ -0,0 +1,34 @@
// This file contains tests for platforms that have no escape
// mechanism for including commas in mount options.
//
// +build darwin
package fuse_test
import (
"runtime"
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
)
func TestMountOptionCommaError(t *testing.T) {
t.Parallel()
// this test is not tied to any specific option, it just needs
// some string content
var evil = "FuseTest,Marker"
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}},
func(conf *fuse.MountConfig) error {
fuse.ForTestSetMountOption(conf, "fusetest", evil)
return nil
},
)
if err == nil {
mnt.Close()
t.Fatal("expected an error about commas")
}
if g, e := err.Error(), `mount options cannot contain commas on `+runtime.GOOS+`: "fusetest"="FuseTest,Marker"`; g != e {
t.Fatalf("wrong error: %q != %q", g, e)
}
}

View File

@ -1,11 +1,15 @@
package fuse_test
import (
"os"
"runtime"
"syscall"
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
)
func init() {
@ -13,6 +17,9 @@ func init() {
}
func TestMountOptionFSName(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("FreeBSD does not support FSName")
}
t.Parallel()
const name = "FuseTestMarker"
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}},
@ -33,6 +40,9 @@ func TestMountOptionFSName(t *testing.T) {
}
func testMountOptionFSNameEvil(t *testing.T, evil string) {
if runtime.GOOS == "freebsd" {
t.Skip("FreeBSD does not support FSName")
}
t.Parallel()
var name = "FuseTest" + evil + "Marker"
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}},
@ -87,6 +97,9 @@ func TestMountOptionSubtype(t *testing.T) {
if runtime.GOOS == "darwin" {
t.Skip("OS X does not support Subtype")
}
if runtime.GOOS == "freebsd" {
t.Skip("FreeBSD does not support Subtype")
}
t.Parallel()
const name = "FuseTestMarker"
mnt, err := fstestutil.MountedT(t, fstestutil.SimpleFS{fstestutil.Dir{}},
@ -139,3 +152,75 @@ func TestMountOptionAllowRootThenAllowOther(t *testing.T) {
t.Fatalf("wrong error: %v != %v", g, e)
}
}
type unwritableFile struct{}
func (f unwritableFile) Attr() fuse.Attr { return fuse.Attr{Mode: 0000} }
func TestMountOptionDefaultPermissions(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("FreeBSD does not support DefaultPermissions")
}
t.Parallel()
mnt, err := fstestutil.MountedT(t,
fstestutil.SimpleFS{
fstestutil.ChildMap{"child": unwritableFile{}},
},
fuse.DefaultPermissions(),
)
if err != nil {
t.Fatal(err)
}
defer mnt.Close()
// This will be prevented by kernel-level access checking when
// DefaultPermissions is used.
f, err := os.OpenFile(mnt.Dir+"/child", os.O_WRONLY, 0000)
if err == nil {
f.Close()
t.Fatal("expected an error")
}
if !os.IsPermission(err) {
t.Fatalf("expected a permission error, got %T: %v", err, err)
}
}
type createrDir struct {
fstestutil.Dir
}
var _ fs.NodeCreater = createrDir{}
func (createrDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
// pick a really distinct error, to identify it later
return nil, nil, fuse.Errno(syscall.ENAMETOOLONG)
}
func TestMountOptionReadOnly(t *testing.T) {
t.Parallel()
mnt, err := fstestutil.MountedT(t,
fstestutil.SimpleFS{createrDir{}},
fuse.ReadOnly(),
)
if err != nil {
t.Fatal(err)
}
defer mnt.Close()
// This will be prevented by kernel-level access checking when
// ReadOnly is used.
f, err := os.Create(mnt.Dir + "/child")
if err == nil {
f.Close()
t.Fatal("expected an error")
}
perr, ok := err.(*os.PathError)
if !ok {
t.Fatalf("expected PathError, got %T: %v", err, err)
}
if perr.Err != syscall.EROFS {
t.Fatalf("expected EROFS, got %T: %v", err, err)
}
}

View File

@ -3,24 +3,24 @@
package syscallx
// This file just contains wrappers for platforms that already have
// the right stuff in stdlib.
// the right stuff in golang.org/x/sys/unix.
import (
"syscall"
"golang.org/x/sys/unix"
)
func Getxattr(path string, attr string, dest []byte) (sz int, err error) {
return syscall.Getxattr(path, attr, dest)
return unix.Getxattr(path, attr, dest)
}
func Listxattr(path string, dest []byte) (sz int, err error) {
return syscall.Listxattr(path, dest)
return unix.Listxattr(path, dest)
}
func Setxattr(path string, attr string, data []byte, flags int) (err error) {
return syscall.Setxattr(path, attr, data, flags)
return unix.Setxattr(path, attr, data, flags)
}
func Removexattr(path string, attr string) (err error) {
return syscall.Removexattr(path, attr)
return unix.Removexattr(path, attr)
}

View File

@ -82,7 +82,7 @@ func init() {
// Use the Airbrake hook to report errors that have Error severity or above to
// an exception tracker. You can create custom hooks, see the Hooks section.
log.AddHook(&logrus_airbrake.AirbrakeHook{})
log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development"))
// Output to stderr instead of stdout, could also be a file.
log.SetOutput(os.Stderr)
@ -164,43 +164,8 @@ You can add hooks for logging levels. For example to send errors to an exception
tracking service on `Error`, `Fatal` and `Panic`, info to StatsD or log to
multiple places simultaneously, e.g. syslog.
```go
// Not the real implementation of the Airbrake hook. Just a simple sample.
import (
log "github.com/Sirupsen/logrus"
)
func init() {
log.AddHook(new(AirbrakeHook))
}
type AirbrakeHook struct{}
// `Fire()` takes the entry that the hook is fired for. `entry.Data[]` contains
// the fields for the entry. See the Fields section of the README.
func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error {
err := airbrake.Notify(entry.Data["error"].(error))
if err != nil {
log.WithFields(log.Fields{
"source": "airbrake",
"endpoint": airbrake.Endpoint,
}).Info("Failed to send error to Airbrake")
}
return nil
}
// `Levels()` returns a slice of `Levels` the hook is fired for.
func (hook *AirbrakeHook) Levels() []log.Level {
return []log.Level{
log.ErrorLevel,
log.FatalLevel,
log.PanicLevel,
}
}
```
Logrus comes with built-in hooks. Add those, or your custom hook, in `init`:
Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in
`init`:
```go
import (
@ -211,7 +176,7 @@ import (
)
func init() {
log.AddHook(new(logrus_airbrake.AirbrakeHook))
log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development"))
hook, err := logrus_syslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, "")
if err != nil {
@ -233,6 +198,9 @@ func init() {
Send errors to remote syslog server.
Uses standard library `log/syslog` behind the scenes.
* [`github.com/Sirupsen/logrus/hooks/bugsnag`](https://github.com/Sirupsen/logrus/blob/master/hooks/bugsnag/bugsnag.go)
Send errors to the Bugsnag exception tracking service.
* [`github.com/nubo/hiprus`](https://github.com/nubo/hiprus)
Send errors to a channel in hipchat.
@ -321,6 +289,11 @@ The built-in logging formatters are:
field to `true`. To force no colored output even if there is a TTY set the
`DisableColors` field to `true`
* `logrus.JSONFormatter`. Logs fields as JSON.
* `logrus_logstash.LogstashFormatter`. Logs fields as Logstash Events (http://logstash.net).
```go
logrus.SetFormatter(&logrus_logstash.LogstashFormatter{Type: “application_name"})
```
Third party logging formatters:

View File

@ -9,6 +9,7 @@ var log = logrus.New()
func init() {
log.Formatter = new(logrus.JSONFormatter)
log.Formatter = new(logrus.TextFormatter) // default
log.Level = logrus.DebugLevel
}
func main() {
@ -23,6 +24,11 @@ func main() {
}
}()
log.WithFields(logrus.Fields{
"animal": "walrus",
"number": 8,
}).Debug("Started observing beach")
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": 10,
@ -33,6 +39,10 @@ func main() {
"number": 122,
}).Warn("The group's number increased tremendously!")
log.WithFields(logrus.Fields{
"temperature": -4,
}).Debug("Temperature changes")
log.WithFields(logrus.Fields{
"animal": "orca",
"size": 9009,

View File

@ -3,21 +3,17 @@ package main
import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus/hooks/airbrake"
"github.com/tobi/airbrake-go"
)
var log = logrus.New()
func init() {
log.Formatter = new(logrus.TextFormatter) // default
log.Hooks.Add(new(logrus_airbrake.AirbrakeHook))
log.Formatter = // default
new(logrus.TextFormatter)
log.Hooks.Add(airbrake.NewHook("https://example.com", "xyz", "development"))
}
func main() {
airbrake.Endpoint = "https://exceptions.whatever.com/notifier_api/v2/notices.xml"
airbrake.ApiKey = "whatever"
airbrake.Environment = "production"
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": 10,

View File

@ -0,0 +1,48 @@
package logstash
import (
"encoding/json"
"fmt"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
"time"
)
// Formatter generates json in logstash format.
// Logstash site: http://logstash.net/
type LogstashFormatter struct {
Type string // if not empty use for logstash type field.
}
func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) {
entry.Data["@version"] = 1
entry.Data["@timestamp"] = entry.Time.Format(time.RFC3339)
// set message field
v, ok := entry.Data["message"]
if ok {
entry.Data["fields.message"] = v
}
entry.Data["message"] = entry.Message
// set level field
v, ok = entry.Data["level"]
if ok {
entry.Data["fields.level"] = v
}
entry.Data["level"] = entry.Level.String()
// set type field
if f.Type != "" {
v, ok = entry.Data["type"]
if ok {
entry.Data["fields.type"] = v
}
entry.Data["type"] = f.Type
}
serialized, err := json.Marshal(entry.Data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
}
return append(serialized, '\n'), nil
}

View File

@ -0,0 +1,52 @@
package logstash
import (
"bytes"
"encoding/json"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
"github.com/stretchr/testify/assert"
"testing"
)
func TestLogstashFormatter(t *testing.T) {
assert := assert.New(t)
lf := LogstashFormatter{Type: "abc"}
fields := logrus.Fields{
"message": "def",
"level": "ijk",
"type": "lmn",
"one": 1,
"pi": 3.14,
"bool": true,
}
entry := logrus.WithFields(fields)
entry.Message = "msg"
entry.Level = logrus.InfoLevel
b, _ := lf.Format(entry)
var data map[string]interface{}
dec := json.NewDecoder(bytes.NewReader(b))
dec.UseNumber()
dec.Decode(&data)
// base fields
assert.Equal(json.Number("1"), data["@version"])
assert.NotEmpty(data["@timestamp"])
assert.Equal("abc", data["type"])
assert.Equal("msg", data["message"])
assert.Equal("info", data["level"])
// substituted fields
assert.Equal("def", data["fields.message"])
assert.Equal("ijk", data["fields.level"])
assert.Equal("lmn", data["fields.type"])
// formats
assert.Equal(json.Number("1"), data["one"])
assert.Equal(json.Number("3.14"), data["pi"])
assert.Equal(true, data["bool"])
}

View File

@ -1,51 +1,51 @@
package logrus_airbrake
package airbrake
import (
"errors"
"fmt"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
"github.com/tobi/airbrake-go"
)
// AirbrakeHook to send exceptions to an exception-tracking service compatible
// with the Airbrake API. You must set:
// * airbrake.Endpoint
// * airbrake.ApiKey
// * airbrake.Environment (only sends exceptions when set to "production")
//
// Before using this hook, to send an error. Entries that trigger an Error,
// Fatal or Panic should now include an "error" field to send to Airbrake.
type AirbrakeHook struct{}
// with the Airbrake API.
type airbrakeHook struct {
APIKey string
Endpoint string
Environment string
}
func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error {
if entry.Data["error"] == nil {
entry.Logger.WithFields(logrus.Fields{
"source": "airbrake",
"endpoint": airbrake.Endpoint,
}).Warn("Exceptions sent to Airbrake must have an 'error' key with the error")
return nil
func NewHook(endpoint, apiKey, env string) *airbrakeHook {
return &airbrakeHook{
APIKey: apiKey,
Endpoint: endpoint,
Environment: env,
}
}
func (hook *airbrakeHook) Fire(entry *logrus.Entry) error {
airbrake.ApiKey = hook.APIKey
airbrake.Endpoint = hook.Endpoint
airbrake.Environment = hook.Environment
var notifyErr error
err, ok := entry.Data["error"].(error)
if !ok {
entry.Logger.WithFields(logrus.Fields{
"source": "airbrake",
"endpoint": airbrake.Endpoint,
}).Warn("Exceptions sent to Airbrake must have an `error` key of type `error`")
return nil
if ok {
notifyErr = err
} else {
notifyErr = errors.New(entry.Message)
}
airErr := airbrake.Notify(err)
airErr := airbrake.Notify(notifyErr)
if airErr != nil {
entry.Logger.WithFields(logrus.Fields{
"source": "airbrake",
"endpoint": airbrake.Endpoint,
"error": airErr,
}).Warn("Failed to send error to Airbrake")
return fmt.Errorf("Failed to send error to Airbrake: %s", airErr)
}
return nil
}
func (hook *AirbrakeHook) Levels() []logrus.Level {
func (hook *airbrakeHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.ErrorLevel,
logrus.FatalLevel,

View File

@ -0,0 +1,133 @@
package airbrake
import (
"encoding/xml"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
)
type notice struct {
Error NoticeError `xml:"error"`
}
type NoticeError struct {
Class string `xml:"class"`
Message string `xml:"message"`
}
type customErr struct {
msg string
}
func (e *customErr) Error() string {
return e.msg
}
const (
testAPIKey = "abcxyz"
testEnv = "development"
expectedClass = "*airbrake.customErr"
expectedMsg = "foo"
unintendedMsg = "Airbrake will not see this string"
)
var (
noticeError = make(chan NoticeError, 1)
)
// TestLogEntryMessageReceived checks if invoking Logrus' log.Error
// method causes an XML payload containing the log entry message is received
// by a HTTP server emulating an Airbrake-compatible endpoint.
func TestLogEntryMessageReceived(t *testing.T) {
log := logrus.New()
ts := startAirbrakeServer(t)
defer ts.Close()
hook := NewHook(ts.URL, testAPIKey, "production")
log.Hooks.Add(hook)
log.Error(expectedMsg)
select {
case received := <-noticeError:
if received.Message != expectedMsg {
t.Errorf("Unexpected message received: %s", received.Message)
}
case <-time.After(time.Second):
t.Error("Timed out; no notice received by Airbrake API")
}
}
// TestLogEntryMessageReceived confirms that, when passing an error type using
// logrus.Fields, a HTTP server emulating an Airbrake endpoint receives the
// error message returned by the Error() method on the error interface
// rather than the logrus.Entry.Message string.
func TestLogEntryWithErrorReceived(t *testing.T) {
log := logrus.New()
ts := startAirbrakeServer(t)
defer ts.Close()
hook := NewHook(ts.URL, testAPIKey, "production")
log.Hooks.Add(hook)
log.WithFields(logrus.Fields{
"error": &customErr{expectedMsg},
}).Error(unintendedMsg)
select {
case received := <-noticeError:
if received.Message != expectedMsg {
t.Errorf("Unexpected message received: %s", received.Message)
}
if received.Class != expectedClass {
t.Errorf("Unexpected error class: %s", received.Class)
}
case <-time.After(time.Second):
t.Error("Timed out; no notice received by Airbrake API")
}
}
// TestLogEntryWithNonErrorTypeNotReceived confirms that, when passing a
// non-error type using logrus.Fields, a HTTP server emulating an Airbrake
// endpoint receives the logrus.Entry.Message string.
//
// Only error types are supported when setting the 'error' field using
// logrus.WithFields().
func TestLogEntryWithNonErrorTypeNotReceived(t *testing.T) {
log := logrus.New()
ts := startAirbrakeServer(t)
defer ts.Close()
hook := NewHook(ts.URL, testAPIKey, "production")
log.Hooks.Add(hook)
log.WithFields(logrus.Fields{
"error": expectedMsg,
}).Error(unintendedMsg)
select {
case received := <-noticeError:
if received.Message != unintendedMsg {
t.Errorf("Unexpected message received: %s", received.Message)
}
case <-time.After(time.Second):
t.Error("Timed out; no notice received by Airbrake API")
}
}
func startAirbrakeServer(t *testing.T) *httptest.Server {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var notice notice
if err := xml.NewDecoder(r.Body).Decode(&notice); err != nil {
t.Error(err)
}
r.Body.Close()
noticeError <- notice.Error
}))
return ts
}

View File

@ -0,0 +1,68 @@
package logrus_bugsnag
import (
"errors"
"github.com/bugsnag/bugsnag-go"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
)
type bugsnagHook struct{}
// ErrBugsnagUnconfigured is returned if NewBugsnagHook is called before
// bugsnag.Configure. Bugsnag must be configured before the hook.
var ErrBugsnagUnconfigured = errors.New("bugsnag must be configured before installing this logrus hook")
// ErrBugsnagSendFailed indicates that the hook failed to submit an error to
// bugsnag. The error was successfully generated, but `bugsnag.Notify()`
// failed.
type ErrBugsnagSendFailed struct {
err error
}
func (e ErrBugsnagSendFailed) Error() string {
return "failed to send error to Bugsnag: " + e.err.Error()
}
// NewBugsnagHook initializes a logrus hook which sends exceptions to an
// exception-tracking service compatible with the Bugsnag API. Before using
// this hook, you must call bugsnag.Configure(). The returned object should be
// registered with a log via `AddHook()`
//
// Entries that trigger an Error, Fatal or Panic should now include an "error"
// field to send to Bugsnag.
func NewBugsnagHook() (*bugsnagHook, error) {
if bugsnag.Config.APIKey == "" {
return nil, ErrBugsnagUnconfigured
}
return &bugsnagHook{}, nil
}
// Fire forwards an error to Bugsnag. Given a logrus.Entry, it extracts the
// "error" field (or the Message if the error isn't present) and sends it off.
func (hook *bugsnagHook) Fire(entry *logrus.Entry) error {
var notifyErr error
err, ok := entry.Data["error"].(error)
if ok {
notifyErr = err
} else {
notifyErr = errors.New(entry.Message)
}
bugsnagErr := bugsnag.Notify(notifyErr)
if bugsnagErr != nil {
return ErrBugsnagSendFailed{bugsnagErr}
}
return nil
}
// Levels enumerates the log levels on which the error should be forwarded to
// bugsnag: everything at or above the "Error" level.
func (hook *bugsnagHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.ErrorLevel,
logrus.FatalLevel,
logrus.PanicLevel,
}
}

View File

@ -0,0 +1,64 @@
package logrus_bugsnag
import (
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/bugsnag/bugsnag-go"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus"
)
type notice struct {
Events []struct {
Exceptions []struct {
Message string `json:"message"`
} `json:"exceptions"`
} `json:"events"`
}
func TestNoticeReceived(t *testing.T) {
msg := make(chan string, 1)
expectedMsg := "foo"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var notice notice
data, _ := ioutil.ReadAll(r.Body)
if err := json.Unmarshal(data, &notice); err != nil {
t.Error(err)
}
_ = r.Body.Close()
msg <- notice.Events[0].Exceptions[0].Message
}))
defer ts.Close()
hook := &bugsnagHook{}
bugsnag.Configure(bugsnag.Configuration{
Endpoint: ts.URL,
ReleaseStage: "production",
APIKey: "12345678901234567890123456789012",
Synchronous: true,
})
log := logrus.New()
log.Hooks.Add(hook)
log.WithFields(logrus.Fields{
"error": errors.New(expectedMsg),
}).Error("Bugsnag will not see this string")
select {
case received := <-msg:
if received != expectedMsg {
t.Errorf("Unexpected message received: %s", received)
}
case <-time.After(time.Second):
t.Error("Timed out; no notice received by Bugsnag API")
}
}

View File

@ -57,5 +57,5 @@ with a call to `NewSentryHook`. This can be changed by assigning a value to the
```go
hook, _ := logrus_sentry.NewSentryHook(...)
hook.Timeout = 20*time.Seconds
hook.Timeout = 20*time.Second
```

View File

@ -11,7 +11,14 @@ type JSONFormatter struct{}
func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) {
data := make(Fields, len(entry.Data)+3)
for k, v := range entry.Data {
data[k] = v
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/Sirupsen/logrus/issues/137
data[k] = v.Error()
default:
data[k] = v
}
}
prefixFieldClashes(data)
data["time"] = entry.Time.Format(time.RFC3339)

View File

@ -0,0 +1,120 @@
package logrus
import (
"encoding/json"
"errors"
"testing"
)
func TestErrorNotLost(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("error", errors.New("wild walrus")))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
entry := make(map[string]interface{})
err = json.Unmarshal(b, &entry)
if err != nil {
t.Fatal("Unable to unmarshal formatted entry: ", err)
}
if entry["error"] != "wild walrus" {
t.Fatal("Error field not set")
}
}
func TestErrorNotLostOnFieldNotNamedError(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("omg", errors.New("wild walrus")))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
entry := make(map[string]interface{})
err = json.Unmarshal(b, &entry)
if err != nil {
t.Fatal("Unable to unmarshal formatted entry: ", err)
}
if entry["omg"] != "wild walrus" {
t.Fatal("Error field not set")
}
}
func TestFieldClashWithTime(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("time", "right now!"))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
entry := make(map[string]interface{})
err = json.Unmarshal(b, &entry)
if err != nil {
t.Fatal("Unable to unmarshal formatted entry: ", err)
}
if entry["fields.time"] != "right now!" {
t.Fatal("fields.time not set to original time field")
}
if entry["time"] != "0001-01-01T00:00:00Z" {
t.Fatal("time field not set to current time, was: ", entry["time"])
}
}
func TestFieldClashWithMsg(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("msg", "something"))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
entry := make(map[string]interface{})
err = json.Unmarshal(b, &entry)
if err != nil {
t.Fatal("Unable to unmarshal formatted entry: ", err)
}
if entry["fields.msg"] != "something" {
t.Fatal("fields.msg not set to original msg field")
}
}
func TestFieldClashWithLevel(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("level", "something"))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
entry := make(map[string]interface{})
err = json.Unmarshal(b, &entry)
if err != nil {
t.Fatal("Unable to unmarshal formatted entry: ", err)
}
if entry["fields.level"] != "something" {
t.Fatal("fields.level not set to original level field")
}
}
func TestJSONEntryEndsWithNewline(t *testing.T) {
formatter := &JSONFormatter{}
b, err := formatter.Format(WithField("level", "something"))
if err != nil {
t.Fatal("Unable to format entry: ", err)
}
if b[len(b)-1] != '\n' {
t.Fatal("Expected JSON log entry to end with a newline")
}
}

View File

@ -65,11 +65,15 @@ func (logger *Logger) WithFields(fields Fields) *Entry {
}
func (logger *Logger) Debugf(format string, args ...interface{}) {
NewEntry(logger).Debugf(format, args...)
if logger.Level >= DebugLevel {
NewEntry(logger).Debugf(format, args...)
}
}
func (logger *Logger) Infof(format string, args ...interface{}) {
NewEntry(logger).Infof(format, args...)
if logger.Level >= InfoLevel {
NewEntry(logger).Infof(format, args...)
}
}
func (logger *Logger) Printf(format string, args ...interface{}) {
@ -77,31 +81,45 @@ func (logger *Logger) Printf(format string, args ...interface{}) {
}
func (logger *Logger) Warnf(format string, args ...interface{}) {
NewEntry(logger).Warnf(format, args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warnf(format, args...)
}
}
func (logger *Logger) Warningf(format string, args ...interface{}) {
NewEntry(logger).Warnf(format, args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warnf(format, args...)
}
}
func (logger *Logger) Errorf(format string, args ...interface{}) {
NewEntry(logger).Errorf(format, args...)
if logger.Level >= ErrorLevel {
NewEntry(logger).Errorf(format, args...)
}
}
func (logger *Logger) Fatalf(format string, args ...interface{}) {
NewEntry(logger).Fatalf(format, args...)
if logger.Level >= FatalLevel {
NewEntry(logger).Fatalf(format, args...)
}
}
func (logger *Logger) Panicf(format string, args ...interface{}) {
NewEntry(logger).Panicf(format, args...)
if logger.Level >= PanicLevel {
NewEntry(logger).Panicf(format, args...)
}
}
func (logger *Logger) Debug(args ...interface{}) {
NewEntry(logger).Debug(args...)
if logger.Level >= DebugLevel {
NewEntry(logger).Debug(args...)
}
}
func (logger *Logger) Info(args ...interface{}) {
NewEntry(logger).Info(args...)
if logger.Level >= InfoLevel {
NewEntry(logger).Info(args...)
}
}
func (logger *Logger) Print(args ...interface{}) {
@ -109,31 +127,45 @@ func (logger *Logger) Print(args ...interface{}) {
}
func (logger *Logger) Warn(args ...interface{}) {
NewEntry(logger).Warn(args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warn(args...)
}
}
func (logger *Logger) Warning(args ...interface{}) {
NewEntry(logger).Warn(args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warn(args...)
}
}
func (logger *Logger) Error(args ...interface{}) {
NewEntry(logger).Error(args...)
if logger.Level >= ErrorLevel {
NewEntry(logger).Error(args...)
}
}
func (logger *Logger) Fatal(args ...interface{}) {
NewEntry(logger).Fatal(args...)
if logger.Level >= FatalLevel {
NewEntry(logger).Fatal(args...)
}
}
func (logger *Logger) Panic(args ...interface{}) {
NewEntry(logger).Panic(args...)
if logger.Level >= PanicLevel {
NewEntry(logger).Panic(args...)
}
}
func (logger *Logger) Debugln(args ...interface{}) {
NewEntry(logger).Debugln(args...)
if logger.Level >= DebugLevel {
NewEntry(logger).Debugln(args...)
}
}
func (logger *Logger) Infoln(args ...interface{}) {
NewEntry(logger).Infoln(args...)
if logger.Level >= InfoLevel {
NewEntry(logger).Infoln(args...)
}
}
func (logger *Logger) Println(args ...interface{}) {
@ -141,21 +173,31 @@ func (logger *Logger) Println(args ...interface{}) {
}
func (logger *Logger) Warnln(args ...interface{}) {
NewEntry(logger).Warnln(args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warnln(args...)
}
}
func (logger *Logger) Warningln(args ...interface{}) {
NewEntry(logger).Warnln(args...)
if logger.Level >= WarnLevel {
NewEntry(logger).Warnln(args...)
}
}
func (logger *Logger) Errorln(args ...interface{}) {
NewEntry(logger).Errorln(args...)
if logger.Level >= ErrorLevel {
NewEntry(logger).Errorln(args...)
}
}
func (logger *Logger) Fatalln(args ...interface{}) {
NewEntry(logger).Fatalln(args...)
if logger.Level >= FatalLevel {
NewEntry(logger).Fatalln(args...)
}
}
func (logger *Logger) Panicln(args ...interface{}) {
NewEntry(logger).Panicln(args...)
if logger.Level >= PanicLevel {
NewEntry(logger).Panicln(args...)
}
}

View File

@ -1,4 +1,3 @@
package logrus
import "syscall"

View File

@ -3,7 +3,6 @@ package logrus
import (
"bytes"
"fmt"
"regexp"
"sort"
"strings"
"time"
@ -15,12 +14,12 @@ const (
green = 32
yellow = 33
blue = 34
gray = 37
)
var (
baseTimestamp time.Time
isTerminal bool
noQuoteNeeded *regexp.Regexp
)
func init() {
@ -46,15 +45,22 @@ type TextFormatter struct {
// Enable logging the full timestamp when a TTY is attached instead of just
// the time passed since beginning of execution.
FullTimestamp bool
// The fields are sorted by default for a consistent output. For applications
// that log extremely frequently and don't use the JSON formatter this may not
// be desired.
DisableSorting bool
}
func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
var keys []string = make([]string, 0, len(entry.Data))
for k := range entry.Data {
keys = append(keys, k)
}
sort.Strings(keys)
if !f.DisableSorting {
sort.Strings(keys)
}
b := &bytes.Buffer{}
@ -82,6 +88,8 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []string) {
var levelColor int
switch entry.Level {
case DebugLevel:
levelColor = gray
case WarnLevel:
levelColor = yellow
case ErrorLevel, FatalLevel, PanicLevel:
@ -107,7 +115,7 @@ func needsQuoting(text string) bool {
for _, ch := range text {
if !((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch < '9') ||
(ch >= '0' && ch <= '9') ||
ch == '-' || ch == '.') {
return false
}

View File

@ -25,9 +25,13 @@ func TestQuoting(t *testing.T) {
checkQuoting(false, "abcd")
checkQuoting(false, "v1.0")
checkQuoting(false, "1234567890")
checkQuoting(true, "/foobar")
checkQuoting(true, "x y")
checkQuoting(true, "x,y")
checkQuoting(false, errors.New("invalid"))
checkQuoting(true, errors.New("invalid argument"))
}
// TODO add tests for sorting etc., this requires a parser for the text
// formatter output.

View File

@ -6,7 +6,7 @@ import (
"runtime"
)
func (logger *Logger) Writer() (*io.PipeWriter) {
func (logger *Logger) Writer() *io.PipeWriter {
reader, writer := io.Pipe()
go logger.writerScanner(reader)

View File

@ -58,6 +58,13 @@ func NewVersion(version string) (*Version, error) {
return &v, nil
}
func Must(v *Version, err error) *Version {
if err != nil {
panic(err)
}
return v
}
func (v *Version) String() string {
var buffer bytes.Buffer

View File

@ -1,7 +1,9 @@
package semver
import (
"errors"
"math/rand"
"reflect"
"testing"
"time"
)
@ -185,3 +187,37 @@ func TestBumpPatch(t *testing.T) {
t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
}
}
func TestMust(t *testing.T) {
tests := []struct {
versionStr string
version *Version
recov interface{}
}{
{
versionStr: "1.0.0",
version: &Version{Major: 1},
},
{
versionStr: "version number",
recov: errors.New("version number is not in dotted-tri format"),
},
}
for _, tt := range tests {
func() {
defer func() {
recov := recover()
if !reflect.DeepEqual(tt.recov, recov) {
t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov)
}
}()
version := Must(NewVersion(tt.versionStr))
if !reflect.DeepEqual(tt.version, version) {
t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version)
}
}()
}
}

View File

@ -1,7 +0,0 @@
language: go
go:
- 1.0
- 1.1
- 1.2
- tip

View File

@ -1,27 +0,0 @@
Copyright (c) 2012 Rodrigo Moraes. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,7 +0,0 @@
context
=======
[![Build Status](https://travis-ci.org/gorilla/context.png?branch=master)](https://travis-ci.org/gorilla/context)
gorilla/context is a general purpose registry for global request variables.
Read the full documentation here: http://www.gorillatoolkit.org/pkg/context

View File

@ -1,143 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context
import (
"net/http"
"sync"
"time"
)
var (
mutex sync.RWMutex
data = make(map[*http.Request]map[interface{}]interface{})
datat = make(map[*http.Request]int64)
)
// Set stores a value for a given key in a given request.
func Set(r *http.Request, key, val interface{}) {
mutex.Lock()
if data[r] == nil {
data[r] = make(map[interface{}]interface{})
datat[r] = time.Now().Unix()
}
data[r][key] = val
mutex.Unlock()
}
// Get returns a value stored for a given key in a given request.
func Get(r *http.Request, key interface{}) interface{} {
mutex.RLock()
if ctx := data[r]; ctx != nil {
value := ctx[key]
mutex.RUnlock()
return value
}
mutex.RUnlock()
return nil
}
// GetOk returns stored value and presence state like multi-value return of map access.
func GetOk(r *http.Request, key interface{}) (interface{}, bool) {
mutex.RLock()
if _, ok := data[r]; ok {
value, ok := data[r][key]
mutex.RUnlock()
return value, ok
}
mutex.RUnlock()
return nil, false
}
// GetAll returns all stored values for the request as a map. Nil is returned for invalid requests.
func GetAll(r *http.Request) map[interface{}]interface{} {
mutex.RLock()
if context, ok := data[r]; ok {
result := make(map[interface{}]interface{}, len(context))
for k, v := range context {
result[k] = v
}
mutex.RUnlock()
return result
}
mutex.RUnlock()
return nil
}
// GetAllOk returns all stored values for the request as a map and a boolean value that indicates if
// the request was registered.
func GetAllOk(r *http.Request) (map[interface{}]interface{}, bool) {
mutex.RLock()
context, ok := data[r]
result := make(map[interface{}]interface{}, len(context))
for k, v := range context {
result[k] = v
}
mutex.RUnlock()
return result, ok
}
// Delete removes a value stored for a given key in a given request.
func Delete(r *http.Request, key interface{}) {
mutex.Lock()
if data[r] != nil {
delete(data[r], key)
}
mutex.Unlock()
}
// Clear removes all values stored for a given request.
//
// This is usually called by a handler wrapper to clean up request
// variables at the end of a request lifetime. See ClearHandler().
func Clear(r *http.Request) {
mutex.Lock()
clear(r)
mutex.Unlock()
}
// clear is Clear without the lock.
func clear(r *http.Request) {
delete(data, r)
delete(datat, r)
}
// Purge removes request data stored for longer than maxAge, in seconds.
// It returns the amount of requests removed.
//
// If maxAge <= 0, all request data is removed.
//
// This is only used for sanity check: in case context cleaning was not
// properly set some request data can be kept forever, consuming an increasing
// amount of memory. In case this is detected, Purge() must be called
// periodically until the problem is fixed.
func Purge(maxAge int) int {
mutex.Lock()
count := 0
if maxAge <= 0 {
count = len(data)
data = make(map[*http.Request]map[interface{}]interface{})
datat = make(map[*http.Request]int64)
} else {
min := time.Now().Unix() - int64(maxAge)
for r := range data {
if datat[r] < min {
clear(r)
count++
}
}
}
mutex.Unlock()
return count
}
// ClearHandler wraps an http.Handler and clears request values at the end
// of a request lifetime.
func ClearHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer Clear(r)
h.ServeHTTP(w, r)
})
}

View File

@ -1,161 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context
import (
"net/http"
"testing"
)
type keyType int
const (
key1 keyType = iota
key2
)
func TestContext(t *testing.T) {
assertEqual := func(val interface{}, exp interface{}) {
if val != exp {
t.Errorf("Expected %v, got %v.", exp, val)
}
}
r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
emptyR, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
// Get()
assertEqual(Get(r, key1), nil)
// Set()
Set(r, key1, "1")
assertEqual(Get(r, key1), "1")
assertEqual(len(data[r]), 1)
Set(r, key2, "2")
assertEqual(Get(r, key2), "2")
assertEqual(len(data[r]), 2)
//GetOk
value, ok := GetOk(r, key1)
assertEqual(value, "1")
assertEqual(ok, true)
value, ok = GetOk(r, "not exists")
assertEqual(value, nil)
assertEqual(ok, false)
Set(r, "nil value", nil)
value, ok = GetOk(r, "nil value")
assertEqual(value, nil)
assertEqual(ok, true)
// GetAll()
values := GetAll(r)
assertEqual(len(values), 3)
// GetAll() for empty request
values = GetAll(emptyR)
if values != nil {
t.Error("GetAll didn't return nil value for invalid request")
}
// GetAllOk()
values, ok = GetAllOk(r)
assertEqual(len(values), 3)
assertEqual(ok, true)
// GetAllOk() for empty request
values, ok = GetAllOk(emptyR)
assertEqual(value, nil)
assertEqual(ok, false)
// Delete()
Delete(r, key1)
assertEqual(Get(r, key1), nil)
assertEqual(len(data[r]), 2)
Delete(r, key2)
assertEqual(Get(r, key2), nil)
assertEqual(len(data[r]), 1)
// Clear()
Clear(r)
assertEqual(len(data), 0)
}
func parallelReader(r *http.Request, key string, iterations int, wait, done chan struct{}) {
<-wait
for i := 0; i < iterations; i++ {
Get(r, key)
}
done <- struct{}{}
}
func parallelWriter(r *http.Request, key, value string, iterations int, wait, done chan struct{}) {
<-wait
for i := 0; i < iterations; i++ {
Get(r, key)
}
done <- struct{}{}
}
func benchmarkMutex(b *testing.B, numReaders, numWriters, iterations int) {
b.StopTimer()
r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
done := make(chan struct{})
b.StartTimer()
for i := 0; i < b.N; i++ {
wait := make(chan struct{})
for i := 0; i < numReaders; i++ {
go parallelReader(r, "test", iterations, wait, done)
}
for i := 0; i < numWriters; i++ {
go parallelWriter(r, "test", "123", iterations, wait, done)
}
close(wait)
for i := 0; i < numReaders+numWriters; i++ {
<-done
}
}
}
func BenchmarkMutexSameReadWrite1(b *testing.B) {
benchmarkMutex(b, 1, 1, 32)
}
func BenchmarkMutexSameReadWrite2(b *testing.B) {
benchmarkMutex(b, 2, 2, 32)
}
func BenchmarkMutexSameReadWrite4(b *testing.B) {
benchmarkMutex(b, 4, 4, 32)
}
func BenchmarkMutex1(b *testing.B) {
benchmarkMutex(b, 2, 8, 32)
}
func BenchmarkMutex2(b *testing.B) {
benchmarkMutex(b, 16, 4, 64)
}
func BenchmarkMutex3(b *testing.B) {
benchmarkMutex(b, 1, 2, 128)
}
func BenchmarkMutex4(b *testing.B) {
benchmarkMutex(b, 128, 32, 256)
}
func BenchmarkMutex5(b *testing.B) {
benchmarkMutex(b, 1024, 2048, 64)
}
func BenchmarkMutex6(b *testing.B) {
benchmarkMutex(b, 2048, 1024, 512)
}

View File

@ -1,82 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package context stores values shared during a request lifetime.
For example, a router can set variables extracted from the URL and later
application handlers can access those values, or it can be used to store
sessions values to be saved at the end of a request. There are several
others common uses.
The idea was posted by Brad Fitzpatrick to the go-nuts mailing list:
http://groups.google.com/group/golang-nuts/msg/e2d679d303aa5d53
Here's the basic usage: first define the keys that you will need. The key
type is interface{} so a key can be of any type that supports equality.
Here we define a key using a custom int type to avoid name collisions:
package foo
import (
"github.com/gorilla/context"
)
type key int
const MyKey key = 0
Then set a variable. Variables are bound to an http.Request object, so you
need a request instance to set a value:
context.Set(r, MyKey, "bar")
The application can later access the variable using the same key you provided:
func MyHandler(w http.ResponseWriter, r *http.Request) {
// val is "bar".
val := context.Get(r, foo.MyKey)
// returns ("bar", true)
val, ok := context.GetOk(r, foo.MyKey)
// ...
}
And that's all about the basic usage. We discuss some other ideas below.
Any type can be stored in the context. To enforce a given type, make the key
private and wrap Get() and Set() to accept and return values of a specific
type:
type key int
const mykey key = 0
// GetMyKey returns a value for this package from the request values.
func GetMyKey(r *http.Request) SomeType {
if rv := context.Get(r, mykey); rv != nil {
return rv.(SomeType)
}
return nil
}
// SetMyKey sets a value for this package in the request values.
func SetMyKey(r *http.Request, val SomeType) {
context.Set(r, mykey, val)
}
Variables must be cleared at the end of a request, to remove all values
that were stored. This can be done in an http.Handler, after a request was
served. Just call Clear() passing the request:
context.Clear(r)
...or use ClearHandler(), which conveniently wraps an http.Handler to clear
variables at the end of a request lifetime.
The Routers from the packages gorilla/mux and gorilla/pat call Clear()
so if you are using either of them you don't need to clear the context manually.
*/
package context

View File

@ -1,7 +0,0 @@
language: go
go:
- 1.0
- 1.1
- 1.2
- tip

View File

@ -1,27 +0,0 @@
Copyright (c) 2012 Rodrigo Moraes. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,7 +0,0 @@
mux
===
[![Build Status](https://travis-ci.org/gorilla/mux.png?branch=master)](https://travis-ci.org/gorilla/mux)
gorilla/mux is a powerful URL router and dispatcher.
Read the full documentation here: http://www.gorillatoolkit.org/pkg/mux

View File

@ -1,21 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"net/http"
"testing"
)
func BenchmarkMux(b *testing.B) {
router := new(Router)
handler := func(w http.ResponseWriter, r *http.Request) {}
router.HandleFunc("/v1/{v1}", handler)
request, _ := http.NewRequest("GET", "/v1/anything", nil)
for i := 0; i < b.N; i++ {
router.ServeHTTP(nil, request)
}
}

View File

@ -1,199 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package gorilla/mux implements a request router and dispatcher.
The name mux stands for "HTTP request multiplexer". Like the standard
http.ServeMux, mux.Router matches incoming requests against a list of
registered routes and calls a handler for the route that matches the URL
or other conditions. The main features are:
* Requests can be matched based on URL host, path, path prefix, schemes,
header and query values, HTTP methods or using custom matchers.
* URL hosts and paths can have variables with an optional regular
expression.
* Registered URLs can be built, or "reversed", which helps maintaining
references to resources.
* Routes can be used as subrouters: nested routes are only tested if the
parent route matches. This is useful to define groups of routes that
share common conditions like a host, a path prefix or other repeated
attributes. As a bonus, this optimizes request matching.
* It implements the http.Handler interface so it is compatible with the
standard http.ServeMux.
Let's start registering a couple of URL paths and handlers:
func main() {
r := mux.NewRouter()
r.HandleFunc("/", HomeHandler)
r.HandleFunc("/products", ProductsHandler)
r.HandleFunc("/articles", ArticlesHandler)
http.Handle("/", r)
}
Here we register three routes mapping URL paths to handlers. This is
equivalent to how http.HandleFunc() works: if an incoming request URL matches
one of the paths, the corresponding handler is called passing
(http.ResponseWriter, *http.Request) as parameters.
Paths can have variables. They are defined using the format {name} or
{name:pattern}. If a regular expression pattern is not defined, the matched
variable will be anything until the next slash. For example:
r := mux.NewRouter()
r.HandleFunc("/products/{key}", ProductHandler)
r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler)
r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler)
The names are used to create a map of route variables which can be retrieved
calling mux.Vars():
vars := mux.Vars(request)
category := vars["category"]
And this is all you need to know about the basic usage. More advanced options
are explained below.
Routes can also be restricted to a domain or subdomain. Just define a host
pattern to be matched. They can also have variables:
r := mux.NewRouter()
// Only matches if domain is "www.domain.com".
r.Host("www.domain.com")
// Matches a dynamic subdomain.
r.Host("{subdomain:[a-z]+}.domain.com")
There are several other matchers that can be added. To match path prefixes:
r.PathPrefix("/products/")
...or HTTP methods:
r.Methods("GET", "POST")
...or URL schemes:
r.Schemes("https")
...or header values:
r.Headers("X-Requested-With", "XMLHttpRequest")
...or query values:
r.Queries("key", "value")
...or to use a custom matcher function:
r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool {
return r.ProtoMajor == 0
})
...and finally, it is possible to combine several matchers in a single route:
r.HandleFunc("/products", ProductsHandler).
Host("www.domain.com").
Methods("GET").
Schemes("http")
Setting the same matching conditions again and again can be boring, so we have
a way to group several routes that share the same requirements.
We call it "subrouting".
For example, let's say we have several URLs that should only match when the
host is "www.domain.com". Create a route for that host and get a "subrouter"
from it:
r := mux.NewRouter()
s := r.Host("www.domain.com").Subrouter()
Then register routes in the subrouter:
s.HandleFunc("/products/", ProductsHandler)
s.HandleFunc("/products/{key}", ProductHandler)
s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler)
The three URL paths we registered above will only be tested if the domain is
"www.domain.com", because the subrouter is tested first. This is not
only convenient, but also optimizes request matching. You can create
subrouters combining any attribute matchers accepted by a route.
Subrouters can be used to create domain or path "namespaces": you define
subrouters in a central place and then parts of the app can register its
paths relatively to a given subrouter.
There's one more thing about subroutes. When a subrouter has a path prefix,
the inner routes use it as base for their paths:
r := mux.NewRouter()
s := r.PathPrefix("/products").Subrouter()
// "/products/"
s.HandleFunc("/", ProductsHandler)
// "/products/{key}/"
s.HandleFunc("/{key}/", ProductHandler)
// "/products/{key}/details"
s.HandleFunc("/{key}/details", ProductDetailsHandler)
Now let's see how to build registered URLs.
Routes can be named. All routes that define a name can have their URLs built,
or "reversed". We define a name calling Name() on a route. For example:
r := mux.NewRouter()
r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).
Name("article")
To build a URL, get the route and call the URL() method, passing a sequence of
key/value pairs for the route variables. For the previous route, we would do:
url, err := r.Get("article").URL("category", "technology", "id", "42")
...and the result will be a url.URL with the following path:
"/articles/technology/42"
This also works for host variables:
r := mux.NewRouter()
r.Host("{subdomain}.domain.com").
Path("/articles/{category}/{id:[0-9]+}").
HandlerFunc(ArticleHandler).
Name("article")
// url.String() will be "http://news.domain.com/articles/technology/42"
url, err := r.Get("article").URL("subdomain", "news",
"category", "technology",
"id", "42")
All variables defined in the route are required, and their values must
conform to the corresponding patterns. These requirements guarantee that a
generated URL will always match a registered route -- the only exception is
for explicitly defined "build-only" routes which never match.
There's also a way to build only the URL host or path for a route:
use the methods URLHost() or URLPath() instead. For the previous route,
we would do:
// "http://news.domain.com/"
host, err := r.Get("article").URLHost("subdomain", "news")
// "/articles/technology/42"
path, err := r.Get("article").URLPath("category", "technology", "id", "42")
And if you use subrouters, host and path defined separately can be built
as well:
r := mux.NewRouter()
s := r.Host("{subdomain}.domain.com").Subrouter()
s.Path("/articles/{category}/{id:[0-9]+}").
HandlerFunc(ArticleHandler).
Name("article")
// "http://news.domain.com/articles/technology/42"
url, err := r.Get("article").URL("subdomain", "news",
"category", "technology",
"id", "42")
*/
package mux

View File

@ -1,353 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"fmt"
"net/http"
"path"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/context"
)
// NewRouter returns a new router instance.
func NewRouter() *Router {
return &Router{namedRoutes: make(map[string]*Route), KeepContext: false}
}
// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
// var router = mux.NewRouter()
//
// func main() {
// http.Handle("/", router)
// }
//
// Or, for Google App Engine, register it in a init() function:
//
// func init() {
// http.Handle("/", router)
// }
//
// This will send all incoming requests to the router.
type Router struct {
// Configurable Handler to be used when no route matches.
NotFoundHandler http.Handler
// Parent route, if this is a subrouter.
parent parentRoute
// Routes to be matched, in order.
routes []*Route
// Routes by name for URL building.
namedRoutes map[string]*Route
// See Router.StrictSlash(). This defines the flag for new routes.
strictSlash bool
// If true, do not clear the request context after handling the request
KeepContext bool
}
// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
for _, route := range r.routes {
if route.Match(req, match) {
return true
}
}
return false
}
// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Clean path to canonical form and redirect.
if p := cleanPath(req.URL.Path); p != req.URL.Path {
// Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query.
// This matches with fix in go 1.2 r.c. 4 for same problem. Go Issue:
// http://code.google.com/p/go/issues/detail?id=5252
url := *req.URL
url.Path = p
p = url.String()
w.Header().Set("Location", p)
w.WriteHeader(http.StatusMovedPermanently)
return
}
var match RouteMatch
var handler http.Handler
if r.Match(req, &match) {
handler = match.Handler
setVars(req, match.Vars)
setCurrentRoute(req, match.Route)
}
if handler == nil {
handler = r.NotFoundHandler
if handler == nil {
handler = http.NotFoundHandler()
}
}
if !r.KeepContext {
defer context.Clear(req)
}
handler.ServeHTTP(w, req)
}
// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
return r.getNamedRoutes()[name]
}
// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
return r.getNamedRoutes()[name]
}
// StrictSlash defines the trailing slash behavior for new routes. The initial
// value is false.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa. In other words, your application will always
// see the path as specified in the route.
//
// When false, if the route path is "/path", accessing "/path/" will not match
// this route and vice versa.
//
// Special case: when a route sets a path prefix using the PathPrefix() method,
// strict slash is ignored for that route because the redirect behavior can't
// be determined from a prefix alone. However, any subrouters created from that
// route inherit the original StrictSlash setting.
func (r *Router) StrictSlash(value bool) *Router {
r.strictSlash = value
return r
}
// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------
// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
if r.namedRoutes == nil {
if r.parent != nil {
r.namedRoutes = r.parent.getNamedRoutes()
} else {
r.namedRoutes = make(map[string]*Route)
}
}
return r.namedRoutes
}
// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
if r.parent != nil {
return r.parent.getRegexpGroup()
}
return nil
}
// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------
// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
route := &Route{parent: r, strictSlash: r.strictSlash}
r.routes = append(r.routes, route)
return route
}
// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
return r.NewRoute().Path(path).Handler(handler)
}
// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
*http.Request)) *Route {
return r.NewRoute().Path(path).HandlerFunc(f)
}
// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
return r.NewRoute().Headers(pairs...)
}
// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
return r.NewRoute().Host(tpl)
}
// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
return r.NewRoute().MatcherFunc(f)
}
// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
return r.NewRoute().Methods(methods...)
}
// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
return r.NewRoute().Path(tpl)
}
// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
return r.NewRoute().PathPrefix(tpl)
}
// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
return r.NewRoute().Queries(pairs...)
}
// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
return r.NewRoute().Schemes(schemes...)
}
// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------
// RouteMatch stores information about a matched route.
type RouteMatch struct {
Route *Route
Handler http.Handler
Vars map[string]string
}
type contextKey int
const (
varsKey contextKey = iota
routeKey
)
// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
if rv := context.Get(r, varsKey); rv != nil {
return rv.(map[string]string)
}
return nil
}
// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
if rv := context.Get(r, routeKey); rv != nil {
return rv.(*Route)
}
return nil
}
func setVars(r *http.Request, val interface{}) {
context.Set(r, varsKey, val)
}
func setCurrentRoute(r *http.Request, val interface{}) {
context.Set(r, routeKey, val)
}
// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------
// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
if p == "" {
return "/"
}
if p[0] != '/' {
p = "/" + p
}
np := path.Clean(p)
// path.Clean removes trailing slash except for root;
// put the trailing slash back if necessary.
if p[len(p)-1] == '/' && np != "/" {
np += "/"
}
return np
}
// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
for _, v1 := range s1 {
for _, v2 := range s2 {
if v1 == v2 {
return fmt.Errorf("mux: duplicated route variable %q", v2)
}
}
}
return nil
}
// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairs(pairs ...string) (map[string]string, error) {
length := len(pairs)
if length%2 != 0 {
return nil, fmt.Errorf(
"mux: number of parameters must be multiple of 2, got %v", pairs)
}
m := make(map[string]string, length/2)
for i := 0; i < length; i += 2 {
m[pairs[i]] = pairs[i+1]
}
return m, nil
}
// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
for _, v := range arr {
if v == value {
return true
}
}
return false
}
// matchMap returns true if the given key/value pairs exist in a given map.
func matchMap(toCheck map[string]string, toMatch map[string][]string,
canonicalKey bool) bool {
for k, v := range toCheck {
// Check if key exists.
if canonicalKey {
k = http.CanonicalHeaderKey(k)
}
if values := toMatch[k]; values == nil {
return false
} else if v != "" {
// If value was defined as an empty string we only check that the
// key exists. Otherwise we also check for equality.
valueExists := false
for _, value := range values {
if v == value {
valueExists = true
break
}
}
if !valueExists {
return false
}
}
}
return true
}

View File

@ -1,943 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"fmt"
"net/http"
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/context"
)
type routeTest struct {
title string // title of the test
route *Route // the route being tested
request *http.Request // a request to test the route
vars map[string]string // the expected vars of the match
host string // the expected host of the match
path string // the expected path of the match
shouldMatch bool // whether the request is expected to match the route at all
shouldRedirect bool // whether the request should result in a redirect
}
func TestHost(t *testing.T) {
// newRequestHost a new request with a method, url, and host header
newRequestHost := func(method, url, host string) *http.Request {
req, err := http.NewRequest(method, url, nil)
if err != nil {
panic(err)
}
req.Host = host
return req
}
tests := []routeTest{
{
title: "Host route match",
route: new(Route).Host("aaa.bbb.ccc"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: true,
},
{
title: "Host route, wrong host in request URL",
route: new(Route).Host("aaa.bbb.ccc"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: false,
},
{
title: "Host route with port, match",
route: new(Route).Host("aaa.bbb.ccc:1234"),
request: newRequest("GET", "http://aaa.bbb.ccc:1234/111/222/333"),
vars: map[string]string{},
host: "aaa.bbb.ccc:1234",
path: "",
shouldMatch: true,
},
{
title: "Host route with port, wrong port in request URL",
route: new(Route).Host("aaa.bbb.ccc:1234"),
request: newRequest("GET", "http://aaa.bbb.ccc:9999/111/222/333"),
vars: map[string]string{},
host: "aaa.bbb.ccc:1234",
path: "",
shouldMatch: false,
},
{
title: "Host route, match with host in request header",
route: new(Route).Host("aaa.bbb.ccc"),
request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc"),
vars: map[string]string{},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: true,
},
{
title: "Host route, wrong host in request header",
route: new(Route).Host("aaa.bbb.ccc"),
request: newRequestHost("GET", "/111/222/333", "aaa.222.ccc"),
vars: map[string]string{},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: false,
},
// BUG {new(Route).Host("aaa.bbb.ccc:1234"), newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:1234"), map[string]string{}, "aaa.bbb.ccc:1234", "", true},
{
title: "Host route with port, wrong host in request header",
route: new(Route).Host("aaa.bbb.ccc:1234"),
request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:9999"),
vars: map[string]string{},
host: "aaa.bbb.ccc:1234",
path: "",
shouldMatch: false,
},
{
title: "Host route with pattern, match",
route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{"v1": "bbb"},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: true,
},
{
title: "Host route with pattern, wrong host in request URL",
route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{"v1": "bbb"},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: false,
},
{
title: "Host route with multiple patterns, match",
route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: true,
},
{
title: "Host route with multiple patterns, wrong host in request URL",
route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"},
host: "aaa.bbb.ccc",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestPath(t *testing.T) {
tests := []routeTest{
{
title: "Path route, match",
route: new(Route).Path("/111/222/333"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{},
host: "",
path: "/111/222/333",
shouldMatch: true,
},
{
title: "Path route, match with trailing slash in request and path",
route: new(Route).Path("/111/"),
request: newRequest("GET", "http://localhost/111/"),
vars: map[string]string{},
host: "",
path: "/111/",
shouldMatch: true,
},
{
title: "Path route, do not match with trailing slash in path",
route: new(Route).Path("/111/"),
request: newRequest("GET", "http://localhost/111"),
vars: map[string]string{},
host: "",
path: "/111",
shouldMatch: false,
},
{
title: "Path route, do not match with trailing slash in request",
route: new(Route).Path("/111"),
request: newRequest("GET", "http://localhost/111/"),
vars: map[string]string{},
host: "",
path: "/111/",
shouldMatch: false,
},
{
title: "Path route, wrong path in request in request URL",
route: new(Route).Path("/111/222/333"),
request: newRequest("GET", "http://localhost/1/2/3"),
vars: map[string]string{},
host: "",
path: "/111/222/333",
shouldMatch: false,
},
{
title: "Path route with pattern, match",
route: new(Route).Path("/111/{v1:[0-9]{3}}/333"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{"v1": "222"},
host: "",
path: "/111/222/333",
shouldMatch: true,
},
{
title: "Path route with pattern, URL in request does not match",
route: new(Route).Path("/111/{v1:[0-9]{3}}/333"),
request: newRequest("GET", "http://localhost/111/aaa/333"),
vars: map[string]string{"v1": "222"},
host: "",
path: "/111/222/333",
shouldMatch: false,
},
{
title: "Path route with multiple patterns, match",
route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"},
host: "",
path: "/111/222/333",
shouldMatch: true,
},
{
title: "Path route with multiple patterns, URL in request does not match",
route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/aaa/333"),
vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"},
host: "",
path: "/111/222/333",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestPathPrefix(t *testing.T) {
tests := []routeTest{
{
title: "PathPrefix route, match",
route: new(Route).PathPrefix("/111"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{},
host: "",
path: "/111",
shouldMatch: true,
},
{
title: "PathPrefix route, match substring",
route: new(Route).PathPrefix("/1"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{},
host: "",
path: "/1",
shouldMatch: true,
},
{
title: "PathPrefix route, URL prefix in request does not match",
route: new(Route).PathPrefix("/111"),
request: newRequest("GET", "http://localhost/1/2/3"),
vars: map[string]string{},
host: "",
path: "/111",
shouldMatch: false,
},
{
title: "PathPrefix route with pattern, match",
route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{"v1": "222"},
host: "",
path: "/111/222",
shouldMatch: true,
},
{
title: "PathPrefix route with pattern, URL prefix in request does not match",
route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/aaa/333"),
vars: map[string]string{"v1": "222"},
host: "",
path: "/111/222",
shouldMatch: false,
},
{
title: "PathPrefix route with multiple patterns, match",
route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/222/333"),
vars: map[string]string{"v1": "111", "v2": "222"},
host: "",
path: "/111/222",
shouldMatch: true,
},
{
title: "PathPrefix route with multiple patterns, URL prefix in request does not match",
route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"),
request: newRequest("GET", "http://localhost/111/aaa/333"),
vars: map[string]string{"v1": "111", "v2": "222"},
host: "",
path: "/111/222",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestHostPath(t *testing.T) {
tests := []routeTest{
{
title: "Host and Path route, match",
route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Host and Path route, wrong host in request URL",
route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
{
title: "Host and Path route with pattern, match",
route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{"v1": "bbb", "v2": "222"},
host: "aaa.bbb.ccc",
path: "/111/222/333",
shouldMatch: true,
},
{
title: "Host and Path route with pattern, URL in request does not match",
route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{"v1": "bbb", "v2": "222"},
host: "aaa.bbb.ccc",
path: "/111/222/333",
shouldMatch: false,
},
{
title: "Host and Path route with multiple patterns, match",
route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"),
request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"),
vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"},
host: "aaa.bbb.ccc",
path: "/111/222/333",
shouldMatch: true,
},
{
title: "Host and Path route with multiple patterns, URL in request does not match",
route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"),
request: newRequest("GET", "http://aaa.222.ccc/111/222/333"),
vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"},
host: "aaa.bbb.ccc",
path: "/111/222/333",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestHeaders(t *testing.T) {
// newRequestHeaders creates a new request with a method, url, and headers
newRequestHeaders := func(method, url string, headers map[string]string) *http.Request {
req, err := http.NewRequest(method, url, nil)
if err != nil {
panic(err)
}
for k, v := range headers {
req.Header.Add(k, v)
}
return req
}
tests := []routeTest{
{
title: "Headers route, match",
route: new(Route).Headers("foo", "bar", "baz", "ding"),
request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "ding"}),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Headers route, bad header values",
route: new(Route).Headers("foo", "bar", "baz", "ding"),
request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "dong"}),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestMethods(t *testing.T) {
tests := []routeTest{
{
title: "Methods route, match GET",
route: new(Route).Methods("GET", "POST"),
request: newRequest("GET", "http://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Methods route, match POST",
route: new(Route).Methods("GET", "POST"),
request: newRequest("POST", "http://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Methods route, bad method",
route: new(Route).Methods("GET", "POST"),
request: newRequest("PUT", "http://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestQueries(t *testing.T) {
tests := []routeTest{
{
title: "Queries route, match",
route: new(Route).Queries("foo", "bar", "baz", "ding"),
request: newRequest("GET", "http://localhost?foo=bar&baz=ding"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route, match with a query string",
route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"),
request: newRequest("GET", "http://www.example.com/api?foo=bar&baz=ding"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route, match with a query string out of order",
route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"),
request: newRequest("GET", "http://www.example.com/api?baz=ding&foo=bar"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route, bad query",
route: new(Route).Queries("foo", "bar", "baz", "ding"),
request: newRequest("GET", "http://localhost?foo=bar&baz=dong"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
{
title: "Queries route with pattern, match",
route: new(Route).Queries("foo", "{v1}"),
request: newRequest("GET", "http://localhost?foo=bar"),
vars: map[string]string{"v1": "bar"},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route with multiple patterns, match",
route: new(Route).Queries("foo", "{v1}", "baz", "{v2}"),
request: newRequest("GET", "http://localhost?foo=bar&baz=ding"),
vars: map[string]string{"v1": "bar", "v2": "ding"},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route with regexp pattern, match",
route: new(Route).Queries("foo", "{v1:[0-9]+}"),
request: newRequest("GET", "http://localhost?foo=10"),
vars: map[string]string{"v1": "10"},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Queries route with regexp pattern, regexp does not match",
route: new(Route).Queries("foo", "{v1:[0-9]+}"),
request: newRequest("GET", "http://localhost?foo=a"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestSchemes(t *testing.T) {
tests := []routeTest{
// Schemes
{
title: "Schemes route, match https",
route: new(Route).Schemes("https", "ftp"),
request: newRequest("GET", "https://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Schemes route, match ftp",
route: new(Route).Schemes("https", "ftp"),
request: newRequest("GET", "ftp://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "Schemes route, bad scheme",
route: new(Route).Schemes("https", "ftp"),
request: newRequest("GET", "http://localhost"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestMatcherFunc(t *testing.T) {
m := func(r *http.Request, m *RouteMatch) bool {
if r.URL.Host == "aaa.bbb.ccc" {
return true
}
return false
}
tests := []routeTest{
{
title: "MatchFunc route, match",
route: new(Route).MatcherFunc(m),
request: newRequest("GET", "http://aaa.bbb.ccc"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: true,
},
{
title: "MatchFunc route, non-match",
route: new(Route).MatcherFunc(m),
request: newRequest("GET", "http://aaa.222.ccc"),
vars: map[string]string{},
host: "",
path: "",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestSubRouter(t *testing.T) {
subrouter1 := new(Route).Host("{v1:[a-z]+}.google.com").Subrouter()
subrouter2 := new(Route).PathPrefix("/foo/{v1}").Subrouter()
tests := []routeTest{
{
route: subrouter1.Path("/{v2:[a-z]+}"),
request: newRequest("GET", "http://aaa.google.com/bbb"),
vars: map[string]string{"v1": "aaa", "v2": "bbb"},
host: "aaa.google.com",
path: "/bbb",
shouldMatch: true,
},
{
route: subrouter1.Path("/{v2:[a-z]+}"),
request: newRequest("GET", "http://111.google.com/111"),
vars: map[string]string{"v1": "aaa", "v2": "bbb"},
host: "aaa.google.com",
path: "/bbb",
shouldMatch: false,
},
{
route: subrouter2.Path("/baz/{v2}"),
request: newRequest("GET", "http://localhost/foo/bar/baz/ding"),
vars: map[string]string{"v1": "bar", "v2": "ding"},
host: "",
path: "/foo/bar/baz/ding",
shouldMatch: true,
},
{
route: subrouter2.Path("/baz/{v2}"),
request: newRequest("GET", "http://localhost/foo/bar"),
vars: map[string]string{"v1": "bar", "v2": "ding"},
host: "",
path: "/foo/bar/baz/ding",
shouldMatch: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
func TestNamedRoutes(t *testing.T) {
r1 := NewRouter()
r1.NewRoute().Name("a")
r1.NewRoute().Name("b")
r1.NewRoute().Name("c")
r2 := r1.NewRoute().Subrouter()
r2.NewRoute().Name("d")
r2.NewRoute().Name("e")
r2.NewRoute().Name("f")
r3 := r2.NewRoute().Subrouter()
r3.NewRoute().Name("g")
r3.NewRoute().Name("h")
r3.NewRoute().Name("i")
if r1.namedRoutes == nil || len(r1.namedRoutes) != 9 {
t.Errorf("Expected 9 named routes, got %v", r1.namedRoutes)
} else if r1.Get("i") == nil {
t.Errorf("Subroute name not registered")
}
}
func TestStrictSlash(t *testing.T) {
r := NewRouter()
r.StrictSlash(true)
tests := []routeTest{
{
title: "Redirect path without slash",
route: r.NewRoute().Path("/111/"),
request: newRequest("GET", "http://localhost/111"),
vars: map[string]string{},
host: "",
path: "/111/",
shouldMatch: true,
shouldRedirect: true,
},
{
title: "Do not redirect path with slash",
route: r.NewRoute().Path("/111/"),
request: newRequest("GET", "http://localhost/111/"),
vars: map[string]string{},
host: "",
path: "/111/",
shouldMatch: true,
shouldRedirect: false,
},
{
title: "Redirect path with slash",
route: r.NewRoute().Path("/111"),
request: newRequest("GET", "http://localhost/111/"),
vars: map[string]string{},
host: "",
path: "/111",
shouldMatch: true,
shouldRedirect: true,
},
{
title: "Do not redirect path without slash",
route: r.NewRoute().Path("/111"),
request: newRequest("GET", "http://localhost/111"),
vars: map[string]string{},
host: "",
path: "/111",
shouldMatch: true,
shouldRedirect: false,
},
{
title: "Propagate StrictSlash to subrouters",
route: r.NewRoute().PathPrefix("/static/").Subrouter().Path("/images/"),
request: newRequest("GET", "http://localhost/static/images"),
vars: map[string]string{},
host: "",
path: "/static/images/",
shouldMatch: true,
shouldRedirect: true,
},
{
title: "Ignore StrictSlash for path prefix",
route: r.NewRoute().PathPrefix("/static/"),
request: newRequest("GET", "http://localhost/static/logo.png"),
vars: map[string]string{},
host: "",
path: "/static/",
shouldMatch: true,
shouldRedirect: false,
},
}
for _, test := range tests {
testRoute(t, test)
}
}
// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------
func getRouteTemplate(route *Route) string {
host, path := "none", "none"
if route.regexp != nil {
if route.regexp.host != nil {
host = route.regexp.host.template
}
if route.regexp.path != nil {
path = route.regexp.path.template
}
}
return fmt.Sprintf("Host: %v, Path: %v", host, path)
}
func testRoute(t *testing.T, test routeTest) {
request := test.request
route := test.route
vars := test.vars
shouldMatch := test.shouldMatch
host := test.host
path := test.path
url := test.host + test.path
shouldRedirect := test.shouldRedirect
var match RouteMatch
ok := route.Match(request, &match)
if ok != shouldMatch {
msg := "Should match"
if !shouldMatch {
msg = "Should not match"
}
t.Errorf("(%v) %v:\nRoute: %#v\nRequest: %#v\nVars: %v\n", test.title, msg, route, request, vars)
return
}
if shouldMatch {
if test.vars != nil && !stringMapEqual(test.vars, match.Vars) {
t.Errorf("(%v) Vars not equal: expected %v, got %v", test.title, vars, match.Vars)
return
}
if host != "" {
u, _ := test.route.URLHost(mapToPairs(match.Vars)...)
if host != u.Host {
t.Errorf("(%v) URLHost not equal: expected %v, got %v -- %v", test.title, host, u.Host, getRouteTemplate(route))
return
}
}
if path != "" {
u, _ := route.URLPath(mapToPairs(match.Vars)...)
if path != u.Path {
t.Errorf("(%v) URLPath not equal: expected %v, got %v -- %v", test.title, path, u.Path, getRouteTemplate(route))
return
}
}
if url != "" {
u, _ := route.URL(mapToPairs(match.Vars)...)
if url != u.Host+u.Path {
t.Errorf("(%v) URL not equal: expected %v, got %v -- %v", test.title, url, u.Host+u.Path, getRouteTemplate(route))
return
}
}
if shouldRedirect && match.Handler == nil {
t.Errorf("(%v) Did not redirect", test.title)
return
}
if !shouldRedirect && match.Handler != nil {
t.Errorf("(%v) Unexpected redirect", test.title)
return
}
}
}
// Tests that the context is cleared or not cleared properly depending on
// the configuration of the router
func TestKeepContext(t *testing.T) {
func1 := func(w http.ResponseWriter, r *http.Request) {}
r := NewRouter()
r.HandleFunc("/", func1).Name("func1")
req, _ := http.NewRequest("GET", "http://localhost/", nil)
context.Set(req, "t", 1)
res := new(http.ResponseWriter)
r.ServeHTTP(*res, req)
if _, ok := context.GetOk(req, "t"); ok {
t.Error("Context should have been cleared at end of request")
}
r.KeepContext = true
req, _ = http.NewRequest("GET", "http://localhost/", nil)
context.Set(req, "t", 1)
r.ServeHTTP(*res, req)
if _, ok := context.GetOk(req, "t"); !ok {
t.Error("Context should NOT have been cleared at end of request")
}
}
type TestA301ResponseWriter struct {
hh http.Header
status int
}
func (ho TestA301ResponseWriter) Header() http.Header {
return http.Header(ho.hh)
}
func (ho TestA301ResponseWriter) Write(b []byte) (int, error) {
return 0, nil
}
func (ho TestA301ResponseWriter) WriteHeader(code int) {
ho.status = code
}
func Test301Redirect(t *testing.T) {
m := make(http.Header)
func1 := func(w http.ResponseWriter, r *http.Request) {}
func2 := func(w http.ResponseWriter, r *http.Request) {}
r := NewRouter()
r.HandleFunc("/api/", func2).Name("func2")
r.HandleFunc("/", func1).Name("func1")
req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil)
res := TestA301ResponseWriter{
hh: m,
status: 0,
}
r.ServeHTTP(&res, req)
if "http://localhost/api/?abc=def" != res.hh["Location"][0] {
t.Errorf("Should have complete URL with query string")
}
}
// https://plus.google.com/101022900381697718949/posts/eWy6DjFJ6uW
func TestSubrouterHeader(t *testing.T) {
expected := "func1 response"
func1 := func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, expected)
}
func2 := func(http.ResponseWriter, *http.Request) {}
r := NewRouter()
s := r.Headers("SomeSpecialHeader", "").Subrouter()
s.HandleFunc("/", func1).Name("func1")
r.HandleFunc("/", func2).Name("func2")
req, _ := http.NewRequest("GET", "http://localhost/", nil)
req.Header.Add("SomeSpecialHeader", "foo")
match := new(RouteMatch)
matched := r.Match(req, match)
if !matched {
t.Errorf("Should match request")
}
if match.Route.GetName() != "func1" {
t.Errorf("Expecting func1 handler, got %s", match.Route.GetName())
}
resp := NewRecorder()
match.Handler.ServeHTTP(resp, req)
if resp.Body.String() != expected {
t.Errorf("Expecting %q", expected)
}
}
// mapToPairs converts a string map to a slice of string pairs
func mapToPairs(m map[string]string) []string {
var i int
p := make([]string, len(m)*2)
for k, v := range m {
p[i] = k
p[i+1] = v
i += 2
}
return p
}
// stringMapEqual checks the equality of two string maps
func stringMapEqual(m1, m2 map[string]string) bool {
nil1 := m1 == nil
nil2 := m2 == nil
if nil1 != nil2 || len(m1) != len(m2) {
return false
}
for k, v := range m1 {
if v != m2[k] {
return false
}
}
return true
}
// newRequest is a helper function to create a new request with a method and url
func newRequest(method, url string) *http.Request {
req, err := http.NewRequest(method, url, nil)
if err != nil {
panic(err)
}
return req
}

View File

@ -1,714 +0,0 @@
// Old tests ported to Go1. This is a mess. Want to drop it one day.
// Copyright 2011 Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"bytes"
"net/http"
"testing"
)
// ----------------------------------------------------------------------------
// ResponseRecorder
// ----------------------------------------------------------------------------
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// ResponseRecorder is an implementation of http.ResponseWriter that
// records its mutations for later inspection in tests.
type ResponseRecorder struct {
Code int // the HTTP response code from WriteHeader
HeaderMap http.Header // the HTTP response headers
Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to
Flushed bool
}
// NewRecorder returns an initialized ResponseRecorder.
func NewRecorder() *ResponseRecorder {
return &ResponseRecorder{
HeaderMap: make(http.Header),
Body: new(bytes.Buffer),
}
}
// DefaultRemoteAddr is the default remote address to return in RemoteAddr if
// an explicit DefaultRemoteAddr isn't set on ResponseRecorder.
const DefaultRemoteAddr = "1.2.3.4"
// Header returns the response headers.
func (rw *ResponseRecorder) Header() http.Header {
return rw.HeaderMap
}
// Write always succeeds and writes to rw.Body, if not nil.
func (rw *ResponseRecorder) Write(buf []byte) (int, error) {
if rw.Body != nil {
rw.Body.Write(buf)
}
if rw.Code == 0 {
rw.Code = http.StatusOK
}
return len(buf), nil
}
// WriteHeader sets rw.Code.
func (rw *ResponseRecorder) WriteHeader(code int) {
rw.Code = code
}
// Flush sets rw.Flushed to true.
func (rw *ResponseRecorder) Flush() {
rw.Flushed = true
}
// ----------------------------------------------------------------------------
func TestRouteMatchers(t *testing.T) {
var scheme, host, path, query, method string
var headers map[string]string
var resultVars map[bool]map[string]string
router := NewRouter()
router.NewRoute().Host("{var1}.google.com").
Path("/{var2:[a-z]+}/{var3:[0-9]+}").
Queries("foo", "bar").
Methods("GET").
Schemes("https").
Headers("x-requested-with", "XMLHttpRequest")
router.NewRoute().Host("www.{var4}.com").
PathPrefix("/foo/{var5:[a-z]+}/{var6:[0-9]+}").
Queries("baz", "ding").
Methods("POST").
Schemes("http").
Headers("Content-Type", "application/json")
reset := func() {
// Everything match.
scheme = "https"
host = "www.google.com"
path = "/product/42"
query = "?foo=bar"
method = "GET"
headers = map[string]string{"X-Requested-With": "XMLHttpRequest"}
resultVars = map[bool]map[string]string{
true: {"var1": "www", "var2": "product", "var3": "42"},
false: {},
}
}
reset2 := func() {
// Everything match.
scheme = "http"
host = "www.google.com"
path = "/foo/product/42/path/that/is/ignored"
query = "?baz=ding"
method = "POST"
headers = map[string]string{"Content-Type": "application/json"}
resultVars = map[bool]map[string]string{
true: {"var4": "google", "var5": "product", "var6": "42"},
false: {},
}
}
match := func(shouldMatch bool) {
url := scheme + "://" + host + path + query
request, _ := http.NewRequest(method, url, nil)
for key, value := range headers {
request.Header.Add(key, value)
}
var routeMatch RouteMatch
matched := router.Match(request, &routeMatch)
if matched != shouldMatch {
// Need better messages. :)
if matched {
t.Errorf("Should match.")
} else {
t.Errorf("Should not match.")
}
}
if matched {
currentRoute := routeMatch.Route
if currentRoute == nil {
t.Errorf("Expected a current route.")
}
vars := routeMatch.Vars
expectedVars := resultVars[shouldMatch]
if len(vars) != len(expectedVars) {
t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars)
}
for name, value := range vars {
if expectedVars[name] != value {
t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars)
}
}
}
}
// 1st route --------------------------------------------------------------
// Everything match.
reset()
match(true)
// Scheme doesn't match.
reset()
scheme = "http"
match(false)
// Host doesn't match.
reset()
host = "www.mygoogle.com"
match(false)
// Path doesn't match.
reset()
path = "/product/notdigits"
match(false)
// Query doesn't match.
reset()
query = "?foo=baz"
match(false)
// Method doesn't match.
reset()
method = "POST"
match(false)
// Header doesn't match.
reset()
headers = map[string]string{}
match(false)
// Everything match, again.
reset()
match(true)
// 2nd route --------------------------------------------------------------
// Everything match.
reset2()
match(true)
// Scheme doesn't match.
reset2()
scheme = "https"
match(false)
// Host doesn't match.
reset2()
host = "sub.google.com"
match(false)
// Path doesn't match.
reset2()
path = "/bar/product/42"
match(false)
// Query doesn't match.
reset2()
query = "?foo=baz"
match(false)
// Method doesn't match.
reset2()
method = "GET"
match(false)
// Header doesn't match.
reset2()
headers = map[string]string{}
match(false)
// Everything match, again.
reset2()
match(true)
}
type headerMatcherTest struct {
matcher headerMatcher
headers map[string]string
result bool
}
var headerMatcherTests = []headerMatcherTest{
{
matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}),
headers: map[string]string{"X-Requested-With": "XMLHttpRequest"},
result: true,
},
{
matcher: headerMatcher(map[string]string{"x-requested-with": ""}),
headers: map[string]string{"X-Requested-With": "anything"},
result: true,
},
{
matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}),
headers: map[string]string{},
result: false,
},
}
type hostMatcherTest struct {
matcher *Route
url string
vars map[string]string
result bool
}
var hostMatcherTests = []hostMatcherTest{
{
matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"),
url: "http://abc.def.ghi/",
vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"},
result: true,
},
{
matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"),
url: "http://a.b.c/",
vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"},
result: false,
},
}
type methodMatcherTest struct {
matcher methodMatcher
method string
result bool
}
var methodMatcherTests = []methodMatcherTest{
{
matcher: methodMatcher([]string{"GET", "POST", "PUT"}),
method: "GET",
result: true,
},
{
matcher: methodMatcher([]string{"GET", "POST", "PUT"}),
method: "POST",
result: true,
},
{
matcher: methodMatcher([]string{"GET", "POST", "PUT"}),
method: "PUT",
result: true,
},
{
matcher: methodMatcher([]string{"GET", "POST", "PUT"}),
method: "DELETE",
result: false,
},
}
type pathMatcherTest struct {
matcher *Route
url string
vars map[string]string
result bool
}
var pathMatcherTests = []pathMatcherTest{
{
matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"),
url: "http://localhost:8080/123/456/789",
vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"},
result: true,
},
{
matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"),
url: "http://localhost:8080/1/2/3",
vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"},
result: false,
},
}
type schemeMatcherTest struct {
matcher schemeMatcher
url string
result bool
}
var schemeMatcherTests = []schemeMatcherTest{
{
matcher: schemeMatcher([]string{"http", "https"}),
url: "http://localhost:8080/",
result: true,
},
{
matcher: schemeMatcher([]string{"http", "https"}),
url: "https://localhost:8080/",
result: true,
},
{
matcher: schemeMatcher([]string{"https"}),
url: "http://localhost:8080/",
result: false,
},
{
matcher: schemeMatcher([]string{"http"}),
url: "https://localhost:8080/",
result: false,
},
}
type urlBuildingTest struct {
route *Route
vars []string
url string
}
var urlBuildingTests = []urlBuildingTest{
{
route: new(Route).Host("foo.domain.com"),
vars: []string{},
url: "http://foo.domain.com",
},
{
route: new(Route).Host("{subdomain}.domain.com"),
vars: []string{"subdomain", "bar"},
url: "http://bar.domain.com",
},
{
route: new(Route).Host("foo.domain.com").Path("/articles"),
vars: []string{},
url: "http://foo.domain.com/articles",
},
{
route: new(Route).Path("/articles"),
vars: []string{},
url: "/articles",
},
{
route: new(Route).Path("/articles/{category}/{id:[0-9]+}"),
vars: []string{"category", "technology", "id", "42"},
url: "/articles/technology/42",
},
{
route: new(Route).Host("{subdomain}.domain.com").Path("/articles/{category}/{id:[0-9]+}"),
vars: []string{"subdomain", "foo", "category", "technology", "id", "42"},
url: "http://foo.domain.com/articles/technology/42",
},
}
func TestHeaderMatcher(t *testing.T) {
for _, v := range headerMatcherTests {
request, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
for key, value := range v.headers {
request.Header.Add(key, value)
}
var routeMatch RouteMatch
result := v.matcher.Match(request, &routeMatch)
if result != v.result {
if v.result {
t.Errorf("%#v: should match %v.", v.matcher, request.Header)
} else {
t.Errorf("%#v: should not match %v.", v.matcher, request.Header)
}
}
}
}
func TestHostMatcher(t *testing.T) {
for _, v := range hostMatcherTests {
request, _ := http.NewRequest("GET", v.url, nil)
var routeMatch RouteMatch
result := v.matcher.Match(request, &routeMatch)
vars := routeMatch.Vars
if result != v.result {
if v.result {
t.Errorf("%#v: should match %v.", v.matcher, v.url)
} else {
t.Errorf("%#v: should not match %v.", v.matcher, v.url)
}
}
if result {
if len(vars) != len(v.vars) {
t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars))
}
for name, value := range vars {
if v.vars[name] != value {
t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value)
}
}
} else {
if len(vars) != 0 {
t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars))
}
}
}
}
func TestMethodMatcher(t *testing.T) {
for _, v := range methodMatcherTests {
request, _ := http.NewRequest(v.method, "http://localhost:8080/", nil)
var routeMatch RouteMatch
result := v.matcher.Match(request, &routeMatch)
if result != v.result {
if v.result {
t.Errorf("%#v: should match %v.", v.matcher, v.method)
} else {
t.Errorf("%#v: should not match %v.", v.matcher, v.method)
}
}
}
}
func TestPathMatcher(t *testing.T) {
for _, v := range pathMatcherTests {
request, _ := http.NewRequest("GET", v.url, nil)
var routeMatch RouteMatch
result := v.matcher.Match(request, &routeMatch)
vars := routeMatch.Vars
if result != v.result {
if v.result {
t.Errorf("%#v: should match %v.", v.matcher, v.url)
} else {
t.Errorf("%#v: should not match %v.", v.matcher, v.url)
}
}
if result {
if len(vars) != len(v.vars) {
t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars))
}
for name, value := range vars {
if v.vars[name] != value {
t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value)
}
}
} else {
if len(vars) != 0 {
t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars))
}
}
}
}
func TestSchemeMatcher(t *testing.T) {
for _, v := range schemeMatcherTests {
request, _ := http.NewRequest("GET", v.url, nil)
var routeMatch RouteMatch
result := v.matcher.Match(request, &routeMatch)
if result != v.result {
if v.result {
t.Errorf("%#v: should match %v.", v.matcher, v.url)
} else {
t.Errorf("%#v: should not match %v.", v.matcher, v.url)
}
}
}
}
func TestUrlBuilding(t *testing.T) {
for _, v := range urlBuildingTests {
u, _ := v.route.URL(v.vars...)
url := u.String()
if url != v.url {
t.Errorf("expected %v, got %v", v.url, url)
/*
reversePath := ""
reverseHost := ""
if v.route.pathTemplate != nil {
reversePath = v.route.pathTemplate.Reverse
}
if v.route.hostTemplate != nil {
reverseHost = v.route.hostTemplate.Reverse
}
t.Errorf("%#v:\nexpected: %q\ngot: %q\nreverse path: %q\nreverse host: %q", v.route, v.url, url, reversePath, reverseHost)
*/
}
}
ArticleHandler := func(w http.ResponseWriter, r *http.Request) {
}
router := NewRouter()
router.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).Name("article")
url, _ := router.Get("article").URL("category", "technology", "id", "42")
expected := "/articles/technology/42"
if url.String() != expected {
t.Errorf("Expected %v, got %v", expected, url.String())
}
}
func TestMatchedRouteName(t *testing.T) {
routeName := "stock"
router := NewRouter()
route := router.NewRoute().Path("/products/").Name(routeName)
url := "http://www.domain.com/products/"
request, _ := http.NewRequest("GET", url, nil)
var rv RouteMatch
ok := router.Match(request, &rv)
if !ok || rv.Route != route {
t.Errorf("Expected same route, got %+v.", rv.Route)
}
retName := rv.Route.GetName()
if retName != routeName {
t.Errorf("Expected %q, got %q.", routeName, retName)
}
}
func TestSubRouting(t *testing.T) {
// Example from docs.
router := NewRouter()
subrouter := router.NewRoute().Host("www.domain.com").Subrouter()
route := subrouter.NewRoute().Path("/products/").Name("products")
url := "http://www.domain.com/products/"
request, _ := http.NewRequest("GET", url, nil)
var rv RouteMatch
ok := router.Match(request, &rv)
if !ok || rv.Route != route {
t.Errorf("Expected same route, got %+v.", rv.Route)
}
u, _ := router.Get("products").URL()
builtUrl := u.String()
// Yay, subroute aware of the domain when building!
if builtUrl != url {
t.Errorf("Expected %q, got %q.", url, builtUrl)
}
}
func TestVariableNames(t *testing.T) {
route := new(Route).Host("{arg1}.domain.com").Path("/{arg1}/{arg2:[0-9]+}")
if route.err == nil {
t.Errorf("Expected error for duplicated variable names")
}
}
func TestRedirectSlash(t *testing.T) {
var route *Route
var routeMatch RouteMatch
r := NewRouter()
r.StrictSlash(false)
route = r.NewRoute()
if route.strictSlash != false {
t.Errorf("Expected false redirectSlash.")
}
r.StrictSlash(true)
route = r.NewRoute()
if route.strictSlash != true {
t.Errorf("Expected true redirectSlash.")
}
route = new(Route)
route.strictSlash = true
route.Path("/{arg1}/{arg2:[0-9]+}/")
request, _ := http.NewRequest("GET", "http://localhost/foo/123", nil)
routeMatch = RouteMatch{}
_ = route.Match(request, &routeMatch)
vars := routeMatch.Vars
if vars["arg1"] != "foo" {
t.Errorf("Expected foo.")
}
if vars["arg2"] != "123" {
t.Errorf("Expected 123.")
}
rsp := NewRecorder()
routeMatch.Handler.ServeHTTP(rsp, request)
if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123/" {
t.Errorf("Expected redirect header.")
}
route = new(Route)
route.strictSlash = true
route.Path("/{arg1}/{arg2:[0-9]+}")
request, _ = http.NewRequest("GET", "http://localhost/foo/123/", nil)
routeMatch = RouteMatch{}
_ = route.Match(request, &routeMatch)
vars = routeMatch.Vars
if vars["arg1"] != "foo" {
t.Errorf("Expected foo.")
}
if vars["arg2"] != "123" {
t.Errorf("Expected 123.")
}
rsp = NewRecorder()
routeMatch.Handler.ServeHTTP(rsp, request)
if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123" {
t.Errorf("Expected redirect header.")
}
}
// Test for the new regexp library, still not available in stable Go.
func TestNewRegexp(t *testing.T) {
var p *routeRegexp
var matches []string
tests := map[string]map[string][]string{
"/{foo:a{2}}": {
"/a": nil,
"/aa": {"aa"},
"/aaa": nil,
"/aaaa": nil,
},
"/{foo:a{2,}}": {
"/a": nil,
"/aa": {"aa"},
"/aaa": {"aaa"},
"/aaaa": {"aaaa"},
},
"/{foo:a{2,3}}": {
"/a": nil,
"/aa": {"aa"},
"/aaa": {"aaa"},
"/aaaa": nil,
},
"/{foo:[a-z]{3}}/{bar:[a-z]{2}}": {
"/a": nil,
"/ab": nil,
"/abc": nil,
"/abcd": nil,
"/abc/ab": {"abc", "ab"},
"/abc/abc": nil,
"/abcd/ab": nil,
},
`/{foo:\w{3,}}/{bar:\d{2,}}`: {
"/a": nil,
"/ab": nil,
"/abc": nil,
"/abc/1": nil,
"/abc/12": {"abc", "12"},
"/abcd/12": {"abcd", "12"},
"/abcd/123": {"abcd", "123"},
},
}
for pattern, paths := range tests {
p, _ = newRouteRegexp(pattern, false, false, false, false)
for path, result := range paths {
matches = p.regexp.FindStringSubmatch(path)
if result == nil {
if matches != nil {
t.Errorf("%v should not match %v.", pattern, path)
}
} else {
if len(matches) != len(result)+1 {
t.Errorf("Expected %v matches, got %v.", len(result)+1, len(matches))
} else {
for k, v := range result {
if matches[k+1] != v {
t.Errorf("Expected %v, got %v.", v, matches[k+1])
}
}
}
}
}
}
}

View File

@ -1,274 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"bytes"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
)
// newRouteRegexp parses a route template and returns a routeRegexp,
// used to match a host, a path or a query string.
//
// It will extract named variables, assemble a regexp to be matched, create
// a "reverse" template to build URLs and compile regexps to validate variable
// values used in URL building.
//
// Previously we accepted only Python-like identifiers for variable
// names ([a-zA-Z_][a-zA-Z0-9_]*), but currently the only restriction is that
// name and pattern can't be empty, and names can't contain a colon.
func newRouteRegexp(tpl string, matchHost, matchPrefix, matchQuery, strictSlash bool) (*routeRegexp, error) {
// Check if it is well-formed.
idxs, errBraces := braceIndices(tpl)
if errBraces != nil {
return nil, errBraces
}
// Backup the original.
template := tpl
// Now let's parse it.
defaultPattern := "[^/]+"
if matchQuery {
defaultPattern = "[^?&]+"
matchPrefix, strictSlash = true, false
} else if matchHost {
defaultPattern = "[^.]+"
matchPrefix, strictSlash = false, false
}
if matchPrefix {
strictSlash = false
}
// Set a flag for strictSlash.
endSlash := false
if strictSlash && strings.HasSuffix(tpl, "/") {
tpl = tpl[:len(tpl)-1]
endSlash = true
}
varsN := make([]string, len(idxs)/2)
varsR := make([]*regexp.Regexp, len(idxs)/2)
pattern := bytes.NewBufferString("")
if !matchQuery {
pattern.WriteByte('^')
}
reverse := bytes.NewBufferString("")
var end int
var err error
for i := 0; i < len(idxs); i += 2 {
// Set all values we are interested in.
raw := tpl[end:idxs[i]]
end = idxs[i+1]
parts := strings.SplitN(tpl[idxs[i]+1:end-1], ":", 2)
name := parts[0]
patt := defaultPattern
if len(parts) == 2 {
patt = parts[1]
}
// Name or pattern can't be empty.
if name == "" || patt == "" {
return nil, fmt.Errorf("mux: missing name or pattern in %q",
tpl[idxs[i]:end])
}
// Build the regexp pattern.
fmt.Fprintf(pattern, "%s(%s)", regexp.QuoteMeta(raw), patt)
// Build the reverse template.
fmt.Fprintf(reverse, "%s%%s", raw)
// Append variable name and compiled pattern.
varsN[i/2] = name
varsR[i/2], err = regexp.Compile(fmt.Sprintf("^%s$", patt))
if err != nil {
return nil, err
}
}
// Add the remaining.
raw := tpl[end:]
pattern.WriteString(regexp.QuoteMeta(raw))
if strictSlash {
pattern.WriteString("[/]?")
}
if !matchPrefix {
pattern.WriteByte('$')
}
reverse.WriteString(raw)
if endSlash {
reverse.WriteByte('/')
}
// Compile full regexp.
reg, errCompile := regexp.Compile(pattern.String())
if errCompile != nil {
return nil, errCompile
}
// Done!
return &routeRegexp{
template: template,
matchHost: matchHost,
matchQuery: matchQuery,
strictSlash: strictSlash,
regexp: reg,
reverse: reverse.String(),
varsN: varsN,
varsR: varsR,
}, nil
}
// routeRegexp stores a regexp to match a host or path and information to
// collect and validate route variables.
type routeRegexp struct {
// The unmodified template.
template string
// True for host match, false for path or query string match.
matchHost bool
// True for query string match, false for path and host match.
matchQuery bool
// The strictSlash value defined on the route, but disabled if PathPrefix was used.
strictSlash bool
// Expanded regexp.
regexp *regexp.Regexp
// Reverse template.
reverse string
// Variable names.
varsN []string
// Variable regexps (validators).
varsR []*regexp.Regexp
}
// Match matches the regexp against the URL host or path.
func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool {
if !r.matchHost {
if r.matchQuery {
return r.regexp.MatchString(req.URL.RawQuery)
} else {
return r.regexp.MatchString(req.URL.Path)
}
}
return r.regexp.MatchString(getHost(req))
}
// url builds a URL part using the given values.
func (r *routeRegexp) url(pairs ...string) (string, error) {
values, err := mapFromPairs(pairs...)
if err != nil {
return "", err
}
urlValues := make([]interface{}, len(r.varsN))
for k, v := range r.varsN {
value, ok := values[v]
if !ok {
return "", fmt.Errorf("mux: missing route variable %q", v)
}
urlValues[k] = value
}
rv := fmt.Sprintf(r.reverse, urlValues...)
if !r.regexp.MatchString(rv) {
// The URL is checked against the full regexp, instead of checking
// individual variables. This is faster but to provide a good error
// message, we check individual regexps if the URL doesn't match.
for k, v := range r.varsN {
if !r.varsR[k].MatchString(values[v]) {
return "", fmt.Errorf(
"mux: variable %q doesn't match, expected %q", values[v],
r.varsR[k].String())
}
}
}
return rv, nil
}
// braceIndices returns the first level curly brace indices from a string.
// It returns an error in case of unbalanced braces.
func braceIndices(s string) ([]int, error) {
var level, idx int
idxs := make([]int, 0)
for i := 0; i < len(s); i++ {
switch s[i] {
case '{':
if level++; level == 1 {
idx = i
}
case '}':
if level--; level == 0 {
idxs = append(idxs, idx, i+1)
} else if level < 0 {
return nil, fmt.Errorf("mux: unbalanced braces in %q", s)
}
}
}
if level != 0 {
return nil, fmt.Errorf("mux: unbalanced braces in %q", s)
}
return idxs, nil
}
// ----------------------------------------------------------------------------
// routeRegexpGroup
// ----------------------------------------------------------------------------
// routeRegexpGroup groups the route matchers that carry variables.
type routeRegexpGroup struct {
host *routeRegexp
path *routeRegexp
queries []*routeRegexp
}
// setMatch extracts the variables from the URL once a route matches.
func (v *routeRegexpGroup) setMatch(req *http.Request, m *RouteMatch, r *Route) {
// Store host variables.
if v.host != nil {
hostVars := v.host.regexp.FindStringSubmatch(getHost(req))
if hostVars != nil {
for k, v := range v.host.varsN {
m.Vars[v] = hostVars[k+1]
}
}
}
// Store path variables.
if v.path != nil {
pathVars := v.path.regexp.FindStringSubmatch(req.URL.Path)
if pathVars != nil {
for k, v := range v.path.varsN {
m.Vars[v] = pathVars[k+1]
}
// Check if we should redirect.
if v.path.strictSlash {
p1 := strings.HasSuffix(req.URL.Path, "/")
p2 := strings.HasSuffix(v.path.template, "/")
if p1 != p2 {
u, _ := url.Parse(req.URL.String())
if p1 {
u.Path = u.Path[:len(u.Path)-1]
} else {
u.Path += "/"
}
m.Handler = http.RedirectHandler(u.String(), 301)
}
}
}
}
// Store query string variables.
rawQuery := req.URL.RawQuery
for _, q := range v.queries {
queryVars := q.regexp.FindStringSubmatch(rawQuery)
if queryVars != nil {
for k, v := range q.varsN {
m.Vars[v] = queryVars[k+1]
}
}
}
}
// getHost tries its best to return the request host.
func getHost(r *http.Request) string {
if !r.URL.IsAbs() {
host := r.Host
// Slice off any port information.
if i := strings.Index(host, ":"); i != -1 {
host = host[:i]
}
return host
}
return r.URL.Host
}

View File

@ -1,524 +0,0 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mux
import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
)
// Route stores information to match a request and build URLs.
type Route struct {
// Parent where the route was registered (a Router).
parent parentRoute
// Request handler for the route.
handler http.Handler
// List of matchers.
matchers []matcher
// Manager for the variables from host and path.
regexp *routeRegexpGroup
// If true, when the path pattern is "/path/", accessing "/path" will
// redirect to the former and vice versa.
strictSlash bool
// If true, this route never matches: it is only used to build URLs.
buildOnly bool
// The name used to build URLs.
name string
// Error resulted from building a route.
err error
}
// Match matches the route against the request.
func (r *Route) Match(req *http.Request, match *RouteMatch) bool {
if r.buildOnly || r.err != nil {
return false
}
// Match everything.
for _, m := range r.matchers {
if matched := m.Match(req, match); !matched {
return false
}
}
// Yay, we have a match. Let's collect some info about it.
if match.Route == nil {
match.Route = r
}
if match.Handler == nil {
match.Handler = r.handler
}
if match.Vars == nil {
match.Vars = make(map[string]string)
}
// Set variables.
if r.regexp != nil {
r.regexp.setMatch(req, match, r)
}
return true
}
// ----------------------------------------------------------------------------
// Route attributes
// ----------------------------------------------------------------------------
// GetError returns an error resulted from building the route, if any.
func (r *Route) GetError() error {
return r.err
}
// BuildOnly sets the route to never match: it is only used to build URLs.
func (r *Route) BuildOnly() *Route {
r.buildOnly = true
return r
}
// Handler --------------------------------------------------------------------
// Handler sets a handler for the route.
func (r *Route) Handler(handler http.Handler) *Route {
if r.err == nil {
r.handler = handler
}
return r
}
// HandlerFunc sets a handler function for the route.
func (r *Route) HandlerFunc(f func(http.ResponseWriter, *http.Request)) *Route {
return r.Handler(http.HandlerFunc(f))
}
// GetHandler returns the handler for the route, if any.
func (r *Route) GetHandler() http.Handler {
return r.handler
}
// Name -----------------------------------------------------------------------
// Name sets the name for the route, used to build URLs.
// If the name was registered already it will be overwritten.
func (r *Route) Name(name string) *Route {
if r.name != "" {
r.err = fmt.Errorf("mux: route already has name %q, can't set %q",
r.name, name)
}
if r.err == nil {
r.name = name
r.getNamedRoutes()[name] = r
}
return r
}
// GetName returns the name for the route, if any.
func (r *Route) GetName() string {
return r.name
}
// ----------------------------------------------------------------------------
// Matchers
// ----------------------------------------------------------------------------
// matcher types try to match a request.
type matcher interface {
Match(*http.Request, *RouteMatch) bool
}
// addMatcher adds a matcher to the route.
func (r *Route) addMatcher(m matcher) *Route {
if r.err == nil {
r.matchers = append(r.matchers, m)
}
return r
}
// addRegexpMatcher adds a host or path matcher and builder to a route.
func (r *Route) addRegexpMatcher(tpl string, matchHost, matchPrefix, matchQuery bool) error {
if r.err != nil {
return r.err
}
r.regexp = r.getRegexpGroup()
if !matchHost && !matchQuery {
if len(tpl) == 0 || tpl[0] != '/' {
return fmt.Errorf("mux: path must start with a slash, got %q", tpl)
}
if r.regexp.path != nil {
tpl = strings.TrimRight(r.regexp.path.template, "/") + tpl
}
}
rr, err := newRouteRegexp(tpl, matchHost, matchPrefix, matchQuery, r.strictSlash)
if err != nil {
return err
}
for _, q := range r.regexp.queries {
if err = uniqueVars(rr.varsN, q.varsN); err != nil {
return err
}
}
if matchHost {
if r.regexp.path != nil {
if err = uniqueVars(rr.varsN, r.regexp.path.varsN); err != nil {
return err
}
}
r.regexp.host = rr
} else {
if r.regexp.host != nil {
if err = uniqueVars(rr.varsN, r.regexp.host.varsN); err != nil {
return err
}
}
if matchQuery {
r.regexp.queries = append(r.regexp.queries, rr)
} else {
r.regexp.path = rr
}
}
r.addMatcher(rr)
return nil
}
// Headers --------------------------------------------------------------------
// headerMatcher matches the request against header values.
type headerMatcher map[string]string
func (m headerMatcher) Match(r *http.Request, match *RouteMatch) bool {
return matchMap(m, r.Header, true)
}
// Headers adds a matcher for request header values.
// It accepts a sequence of key/value pairs to be matched. For example:
//
// r := mux.NewRouter()
// r.Headers("Content-Type", "application/json",
// "X-Requested-With", "XMLHttpRequest")
//
// The above route will only match if both request header values match.
//
// It the value is an empty string, it will match any value if the key is set.
func (r *Route) Headers(pairs ...string) *Route {
if r.err == nil {
var headers map[string]string
headers, r.err = mapFromPairs(pairs...)
return r.addMatcher(headerMatcher(headers))
}
return r
}
// Host -----------------------------------------------------------------------
// Host adds a matcher for the URL host.
// It accepts a template with zero or more URL variables enclosed by {}.
// Variables can define an optional regexp pattern to me matched:
//
// - {name} matches anything until the next dot.
//
// - {name:pattern} matches the given regexp pattern.
//
// For example:
//
// r := mux.NewRouter()
// r.Host("www.domain.com")
// r.Host("{subdomain}.domain.com")
// r.Host("{subdomain:[a-z]+}.domain.com")
//
// Variable names must be unique in a given route. They can be retrieved
// calling mux.Vars(request).
func (r *Route) Host(tpl string) *Route {
r.err = r.addRegexpMatcher(tpl, true, false, false)
return r
}
// MatcherFunc ----------------------------------------------------------------
// MatcherFunc is the function signature used by custom matchers.
type MatcherFunc func(*http.Request, *RouteMatch) bool
func (m MatcherFunc) Match(r *http.Request, match *RouteMatch) bool {
return m(r, match)
}
// MatcherFunc adds a custom function to be used as request matcher.
func (r *Route) MatcherFunc(f MatcherFunc) *Route {
return r.addMatcher(f)
}
// Methods --------------------------------------------------------------------
// methodMatcher matches the request against HTTP methods.
type methodMatcher []string
func (m methodMatcher) Match(r *http.Request, match *RouteMatch) bool {
return matchInArray(m, r.Method)
}
// Methods adds a matcher for HTTP methods.
// It accepts a sequence of one or more methods to be matched, e.g.:
// "GET", "POST", "PUT".
func (r *Route) Methods(methods ...string) *Route {
for k, v := range methods {
methods[k] = strings.ToUpper(v)
}
return r.addMatcher(methodMatcher(methods))
}
// Path -----------------------------------------------------------------------
// Path adds a matcher for the URL path.
// It accepts a template with zero or more URL variables enclosed by {}. The
// template must start with a "/".
// Variables can define an optional regexp pattern to me matched:
//
// - {name} matches anything until the next slash.
//
// - {name:pattern} matches the given regexp pattern.
//
// For example:
//
// r := mux.NewRouter()
// r.Path("/products/").Handler(ProductsHandler)
// r.Path("/products/{key}").Handler(ProductsHandler)
// r.Path("/articles/{category}/{id:[0-9]+}").
// Handler(ArticleHandler)
//
// Variable names must be unique in a given route. They can be retrieved
// calling mux.Vars(request).
func (r *Route) Path(tpl string) *Route {
r.err = r.addRegexpMatcher(tpl, false, false, false)
return r
}
// PathPrefix -----------------------------------------------------------------
// PathPrefix adds a matcher for the URL path prefix. This matches if the given
// template is a prefix of the full URL path. See Route.Path() for details on
// the tpl argument.
//
// Note that it does not treat slashes specially ("/foobar/" will be matched by
// the prefix "/foo") so you may want to use a trailing slash here.
//
// Also note that the setting of Router.StrictSlash() has no effect on routes
// with a PathPrefix matcher.
func (r *Route) PathPrefix(tpl string) *Route {
r.err = r.addRegexpMatcher(tpl, false, true, false)
return r
}
// Query ----------------------------------------------------------------------
// Queries adds a matcher for URL query values.
// It accepts a sequence of key/value pairs. Values may define variables.
// For example:
//
// r := mux.NewRouter()
// r.Queries("foo", "bar", "id", "{id:[0-9]+}")
//
// The above route will only match if the URL contains the defined queries
// values, e.g.: ?foo=bar&id=42.
//
// It the value is an empty string, it will match any value if the key is set.
//
// Variables can define an optional regexp pattern to me matched:
//
// - {name} matches anything until the next slash.
//
// - {name:pattern} matches the given regexp pattern.
func (r *Route) Queries(pairs ...string) *Route {
length := len(pairs)
if length%2 != 0 {
r.err = fmt.Errorf(
"mux: number of parameters must be multiple of 2, got %v", pairs)
return nil
}
for i := 0; i < length; i += 2 {
if r.err = r.addRegexpMatcher(pairs[i]+"="+pairs[i+1], false, true, true); r.err != nil {
return r
}
}
return r
}
// Schemes --------------------------------------------------------------------
// schemeMatcher matches the request against URL schemes.
type schemeMatcher []string
func (m schemeMatcher) Match(r *http.Request, match *RouteMatch) bool {
return matchInArray(m, r.URL.Scheme)
}
// Schemes adds a matcher for URL schemes.
// It accepts a sequence of schemes to be matched, e.g.: "http", "https".
func (r *Route) Schemes(schemes ...string) *Route {
for k, v := range schemes {
schemes[k] = strings.ToLower(v)
}
return r.addMatcher(schemeMatcher(schemes))
}
// Subrouter ------------------------------------------------------------------
// Subrouter creates a subrouter for the route.
//
// It will test the inner routes only if the parent route matched. For example:
//
// r := mux.NewRouter()
// s := r.Host("www.domain.com").Subrouter()
// s.HandleFunc("/products/", ProductsHandler)
// s.HandleFunc("/products/{key}", ProductHandler)
// s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler)
//
// Here, the routes registered in the subrouter won't be tested if the host
// doesn't match.
func (r *Route) Subrouter() *Router {
router := &Router{parent: r, strictSlash: r.strictSlash}
r.addMatcher(router)
return router
}
// ----------------------------------------------------------------------------
// URL building
// ----------------------------------------------------------------------------
// URL builds a URL for the route.
//
// It accepts a sequence of key/value pairs for the route variables. For
// example, given this route:
//
// r := mux.NewRouter()
// r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).
// Name("article")
//
// ...a URL for it can be built using:
//
// url, err := r.Get("article").URL("category", "technology", "id", "42")
//
// ...which will return an url.URL with the following path:
//
// "/articles/technology/42"
//
// This also works for host variables:
//
// r := mux.NewRouter()
// r.Host("{subdomain}.domain.com").
// HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).
// Name("article")
//
// // url.String() will be "http://news.domain.com/articles/technology/42"
// url, err := r.Get("article").URL("subdomain", "news",
// "category", "technology",
// "id", "42")
//
// All variables defined in the route are required, and their values must
// conform to the corresponding patterns.
func (r *Route) URL(pairs ...string) (*url.URL, error) {
if r.err != nil {
return nil, r.err
}
if r.regexp == nil {
return nil, errors.New("mux: route doesn't have a host or path")
}
var scheme, host, path string
var err error
if r.regexp.host != nil {
// Set a default scheme.
scheme = "http"
if host, err = r.regexp.host.url(pairs...); err != nil {
return nil, err
}
}
if r.regexp.path != nil {
if path, err = r.regexp.path.url(pairs...); err != nil {
return nil, err
}
}
return &url.URL{
Scheme: scheme,
Host: host,
Path: path,
}, nil
}
// URLHost builds the host part of the URL for a route. See Route.URL().
//
// The route must have a host defined.
func (r *Route) URLHost(pairs ...string) (*url.URL, error) {
if r.err != nil {
return nil, r.err
}
if r.regexp == nil || r.regexp.host == nil {
return nil, errors.New("mux: route doesn't have a host")
}
host, err := r.regexp.host.url(pairs...)
if err != nil {
return nil, err
}
return &url.URL{
Scheme: "http",
Host: host,
}, nil
}
// URLPath builds the path part of the URL for a route. See Route.URL().
//
// The route must have a path defined.
func (r *Route) URLPath(pairs ...string) (*url.URL, error) {
if r.err != nil {
return nil, r.err
}
if r.regexp == nil || r.regexp.path == nil {
return nil, errors.New("mux: route doesn't have a path")
}
path, err := r.regexp.path.url(pairs...)
if err != nil {
return nil, err
}
return &url.URL{
Path: path,
}, nil
}
// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------
// parentRoute allows routes to know about parent host and path definitions.
type parentRoute interface {
getNamedRoutes() map[string]*Route
getRegexpGroup() *routeRegexpGroup
}
// getNamedRoutes returns the map where named routes are registered.
func (r *Route) getNamedRoutes() map[string]*Route {
if r.parent == nil {
// During tests router is not always set.
r.parent = NewRouter()
}
return r.parent.getNamedRoutes()
}
// getRegexpGroup returns regexp definitions from this route.
func (r *Route) getRegexpGroup() *routeRegexpGroup {
if r.regexp == nil {
if r.parent == nil {
// During tests router is not always set.
r.parent = NewRouter()
}
regexp := r.parent.getRegexpGroup()
if regexp == nil {
r.regexp = new(routeRegexpGroup)
} else {
// Copy.
r.regexp = &routeRegexpGroup{
host: regexp.host,
path: regexp.path,
queries: regexp.queries,
}
}
}
return r.regexp
}

View File

@ -94,3 +94,37 @@ func TestBase58(t *testing.T) {
}
}
}
func BenchmarkDecodeShort(b *testing.B) {
const in = "1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = Decode(in)
}
}
func BenchmarkEncodeShort(b *testing.B) {
var in = []byte("00eb15231dfceb60925886b67d065299925915aeb172c06647")
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = Encode(in)
}
}
func BenchmarkDecodeOneKilo(b *testing.B) {
const in = "3GimCffBLAHhXMCeNxX2nST6dBem9pbUi3KVKykW73LmewcFtMk9oh9eNPdNR2eSzNqp7Z3E21vrWUkGHzJ7w2yqDUDJ4LKo1w5D6aafZ4SUoNQyrSVxyVG3pwgoZkKXMZVixRyiPZVUpekrsTvZuUoW7mB6BQgDTXbDuMMSRoNR7yiUTKpgwTD61DLmhNZopNxfFjn4avpYPgzsTB94iWueq1yU3EoruWCUMvp6fc1CEbDrZY3pkx9oUbUaSMC37rruBKSSGHh1ZE3XK3kQXBCFraMmUQf8dagofMEg5aTnDiLAZjLyWJMdnQwW1FqKKztP8KAQS2JX8GCCfc68KB4VGf2CfEGXtaapnsNWFrHuWi7Wo5vqyuHd21zGm1u5rsiR6tKNCsFC4nzf3WUNxJNoZrDSdF9KERqhTWWmmcM4qdKRCtBWKTrs1DJD2oiK6BK9BgwoW2dfQdKuxojFyFvmxqPKDDAEZPPpJ51wHoFzBFMM1tUBBkN15cT2GpNwKzDcjHPKJAQ6FNRgppfQytzqpq76sSeZaWAB8hhULMJCQGU57ZUjvP7xYAQwtACBnYrjdxA91XwXFbq5AsQJwAmLw6euKVWNyv11BuHrejVmnNViWg5kuZBrtgL6NtzRWHtdxngHDMtuyky3brqGXaGQhUyXrkSpeknkkHL6NLThHH5NPnfFMVPwn2xf5UM5R51X2nTBzADSVcpi4cT7i44dT7o3yRKWtKfUzZiuNyTcSSrfH8KVdLap5ZKLmdPuXM65M2Z5wJVh3Uc4iv6iZKk44RKikM7zs1hqC4sBxRwLZjxhKvvMXDjDcYFkzyUkues4y7fjdCnVTxc4vTYUqcbY2k2WMssyj9SDseVc7dVrEvWCLQtYy79mJFoz1Hmsfk5ynE28ipznzQ3yTBugLHA6j6qW3S74eY4pJ6iynFEuXT4RqqkLGFcMh3goqS7CkphUMzP4wuJyGnzqa5tCno4U3dJ2jUL7Povg8voRqYAfiHyXC8zuhn225EdmRcSnu2pAuutQVV9hN3bkjfzAFUhUWKki8SwXtFSjy6NJyrYUiaze4p7ApsjHQBCgg2zAoBaGCwVN8991Jny31B5vPyYHy1oRSE4xTVZ7tTw9FyQ7w9p1NSEF4sziCxZHh5rFWZKAajc5c7KaMNDvHPNV6S62MTFGTyuKPQNbv9yHRGN4eH6SnZGW6snvEVdYCspWZ1U3Nbxo6vCmBK95UyYpcxHgg1CCGdU4s3edju2NDQkMifyPkJdkabzzHVDhJJbChAJc1ACQfNW74VXXwrBZmeZyA2R28MBctDyXuSuffiwueys2LVowLu9wiTHUox7KQjtHK2c9howk9czzx2mpnYzkVYH42CYsWa5514EM4CJEXPJSSbXSgJJ"
b.SetBytes(int64(len(in))) // 1024
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = Decode(in)
}
}
func BenchmarkEncodeOneKilo(b *testing.B) {
var in = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x00\x00\x04\xff\xfb\x63\xc9\x7e\x5f\x97\x68\xe5\x10\x08\xe5\xa5\x9a\x7c\x24\x75\x35\xbe\xaf\x37\x0b\xc3\xf4\x18\x62\x4a\xb7\x18\xd0\x10\x0b\x6b\x62\xe1\x36\x0f\x62\xa6\xeb\x78\xa5\xf3\x33\x52\xbc\xdf\x04\xcb\x37\xcf\x3d\x97\x5c\xb8\x75\x09\x1f\x18\x9f\xfc\xa9\xda\x1e\x59\x77\x09\x9c\x5d\xb6\xf2\x9e\x45\xb7\x5e\x5d\x11\xf1\x20\x14\x85\xf8\x54\x87\x8c\x1e\x2c\x2e\x15\x57\x89\xe7\x5d\x49\xb6\xae\x24\x3a\x20\x50\x0e\xa7\x5b\x10\xbf\x0a\xb4\x01\x42\xed\xce\x2d\x45\x21\xb6\xe8\x64\x73\x4e\x7e\x0a\x36\x1d\x57\x0a\x5e\x1c\x21\xc2\xb8\xe7\x89\x82\xe4\x04\x7e\x50\xff\xda\x4f\xfe\x11\x95\xfb\x35\xf9\x6d\x32\xce\xef\x8f\x3d\x1b\xdb\x38\xfa\xcd\x26\x36\x12\x93\xa0\x96\xea\x42\xbe\xd6\x85\x86\xc1\xc1\xe2\x55\x41\xd1\x7f\x8d\x0e\x00\x81\x58\xb4\x10\xbb\x64\x92\x05\x07\xa9\xd5\xd9\x40\x28\x8b\x9b\x4c\x8d\x8e\x4e\x69\xf9\xc9\x35\xea\xda\x2f\x61\x87\x35\x2d\x6b\x25\x32\xf0\x7e\x89\x1a\xcb\xc0\xea\x66\x88\x99\x39\xe0\x3b\x24\x3b\x05\x74\xd3\x72\xf6\x48\x15\xdc\x02\x0a\xbf\xc8\x49\x42\x10\x22\xeb\xe9\x44\x71\x55\xaf\x67\x67\xe6\x2a\x40\x31\x81\xb9\x6f\x65\x86\x0f\x0f\x9d\x58\x4c\x51\xc1\x2e\x4e\x60\x7e\xe8\x93\x39\x90\xda\xe5\xbe\xec\xe4\xdd\xbc\x1d\xba\x40\xa6\x85\xd9\xb2\xec\xb4\x26\x74\xee\xc1\xec\xe3\x40\xb9\x49\xa3\xe1\x26\x76\x8a\xeb\x95\xc8\x72\xb0\x85\x36\x19\x3f\x55\x06\x7b\xcd\x3e\xd0\xdf\x7e\x8d\x2a\xea\xa6\x24\xc6\xf6\xfb\xda\xe0\x45\xcf\x32\x0e\xbc\xf4\x41\x7d\x71\x3d\x86\xf9\xb4\xaf\x07\xa0\xd1\x34\x8a\x02\x28\x56\xd4\xcc\x36\x44\x98\x44\xcb\x9d\xc5\xfc\x45\x2d\xc4\x5c\xfe\xce\xaa\x44\xda\x66\x52\x2d\x32\x6e\x13\x32\xac\xaf\x13\x72\x87\x79\xd2\x92\x54\x9f\xc7\xb9\xf3\x21\xae\xdd\x69\x44\xe9\x46\x94\x1c\x62\x84\x03\xe0\xbf\x66\xfb\xe0\x79\xf9\x57\x9e\x22\x9e\x23\x2d\x2a\x73\xeb\x74\x38\xf0\xea\x5d\xb3\x8f\x87\x26\x3e\x3c\x54\x11\xb7\x98\xbd\x7f\x78\x64\xa3\xf1\x8f\xa9\x5e\x4f\x18\x3f\xa7\x1f\x3a\x29\x27\x27\xb7\x49\x40\x16\x18\x1f\xd3\xed\x86\x61\xbd\xc3\x4e\x4a\x53\x37\x78\x5c\x00\xd3\x50\x45\x1c\x55\xc0\x9b\xd7\x62\x29\x88\x2e\xa4\x0d\x6a\x15\x6c\x33\x3c\xe7\x31\xfa\xc1\xaf\xdf\x7a\x3e\x37\x3e\xe5\xbc\xfd\xfb\x9b\x72\x10\x35\x90\x25\x6e\x87\x0d\x74\x1c\xfd\xe3\x0b\xee\xf5\x92\x28\x8d\x22\x8a\x49\x7b\xcd\xbb\xd8\x24\x6b\x5e\x58\x40\xec\x1b\x6c\xed\x8e\xcb\x56\x62\xa6\xb4\x42\x3d\x7d\xa2\xef\x27\x27\x46\x50\xbc\x5e\x37\x9b\x27\x72\xf0\xea\xa7\xe7\x4d\xf4\xae\x7e\x95\x8f\x91\x2e\x58\xc4\x6a\x06\xda\x7a\x06\x5c\x8d\xfe\xef\xf5\xb3\x0f\xb4\x0a\x20\x53\xd8\x35\x80\x02\xca\x97\x81\xb6\x1c\x4b\x8f\xb7\xee\xd0\xc3\x88\x6c\x76\x3e\xb0\x28\xce\xa1\x9f\x76\x5f\xaa\xc3\x53\x44\x09\x70\xa3\x95\xd9\x8c\x54\xba\x8a\x9a\x6b\xce\xc3\x07\xdf\x13\x6d\xea\x0f\x51\x9c\xe2\x81\x87\xf6\x82\x7a\x70\xd8\xfa\xe2\xa8\x32\xc1\x5e\x53\xc2\x85\xe9\x61\x8a\x17\x82\x12\xab\x92\x79\x2b\xed\x07\xca\x1e\x93\x23\x9c\x4b\xd2\x89\x86\xac\x55\xf9\x50\x23\x8f\x9e\xd3\xab\x22\x57\x91\x5a\x0b\x48\xd7\xa2\xb8\x06\xbb\x74\xae\xe9\xca\x06\x41\x8d\x6a\x00\x42\xc4\x40\xa9\xfe\xae\x88\x42\xc2\x83\xe0\x8a\xd8\x5c\xbb\x5a\xb5\x9c\x1d\xa5\xbe\x67\x50\xb1\x4e\xec\x96\x65\xaa\x87\x5b\xb0\x76\x88\xe3\x1b\xcb\x38\x21\x02\x8e\xc9\xe7\xf5\xc7\xe1\x1d\xe8\xeb\x54\x0e\x0b\xea\xd1\x2e\xad\xbb\xec\x22\x21\xb3\x64\x36\x29\x34\x5e\x3a\x22\xe8\x03\x4b\x86\xb1\x67\x7d\x4f\x48\x6d\xfb\x4b\xde\xe6\x4c\xb0\xaf\x40\x66\xab\xe9\x1a\x4e\xae\x1a\x7e\x05\xc5\x67\x2a\x95\x6d\xc2\x61\x35\x20\xfe\x33\xc3\x2c\x7f\x9b\xbe\x9f\x9a\xd5\xf0\x63\x28\xa1\x94\xb1\x5c\xc1\x18\x6b\x5b\x33\xb4\x4d\xcf\xbe\xf7\xb2\x94\x58\xaa\xcf\xad\xc8\x75\x93\x1a\x08\xf4\xd2\xd9\xf6\x95\x03\x3b\xf3\x4e\xfb\x15\xe4\x28\xed\xd5\x79\xd9\xbf\xb7\x8f\xb2\x70\x16\x4c\x2d\x65\xf6\xec\x33\x1e\xaf\xea\x46\x69\xc6\x9a\x6b\xdd\xf3\x57\xe0\x1d\x28\xcd\xf8\x83\x3d\x94\x4c\x2f\x6e\xfd\x51\x3d\xa8\xff\xcb\x33\xad\x32\x42\x0e\xd3\x00\x0a\xe5\x71\x76\x3b\x83\xc9\x2a\x67\x50\xc3\xa5\xeb\x4d\x8d\x67\xd6\xd9\x1b\x9a\x5a\xbe\xdd\xc5\x15\x00\xcf\x97\x0f\x47\x44\x34\x1d\x4e\xb6\x6f\x91\x31\xf3\x45\x0f\x59\x48\x10\x23\x53\x40\x49\x83\xe6\xc8\xdf\x51\x6c\xa8\x9f\x3a\x43\x3d\xb9\xd4\xea\x30\x4d\xe0\xd2\xb8\x44\xf3\x91\x20\x79\xdb\x7b\xe6\x50\xf9\x0f\xfb\x4c\xac\x79\x93\xf6\xf8\x96\x0d\x55\x7c\x41\x9b\x1a\x86\xad\x4b\xd1\xf9\x5d\xed\x3a\x4f\xc9\x64\x72\xd4\x22\x53\x59\x2f\x01\x00\x00\xff\xff\xc6\xfd\xa0\x37\x00\x04\x00\x00")
b.SetBytes(int64(len(in))) // 1024
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = Encode(in)
}
}

View File

@ -1,4 +1,5 @@
// +build darwin freebsd dragonfly netbsd openbsd linux
package reuseport
import (

View File

@ -1,8 +1,6 @@
package reuseport
import (
"net"
)
import "net"
// TODO. for now, just pass it over to net.Listen/net.Dial
@ -10,6 +8,14 @@ func listen(network, address string) (net.Listener, error) {
return net.Listen(network, address)
}
func listenPacket(netw, laddr string) (net.PacketConn, error) {
return net.ListenPacket(netw, laddr)
}
func listenStream(netw, addr string) (net.Listener, error) {
return listen(netw, addr)
}
func dial(dialer net.Dialer, network, address string) (net.Conn, error) {
return dialer.Dial(network, address)
}

View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package reuseport

View File

@ -1,4 +1,4 @@
// +build darwin freebsd dragonfly netbsd openbsd
// +build darwin,amd64 freebsd dragonfly netbsd openbsd
package poll

View File

@ -0,0 +1,57 @@
package poll
import (
"syscall"
"time"
)
type Poller struct {
kqfd int
event syscall.Kevent_t
}
func New(fd int) (p *Poller, err error) {
p = &Poller{}
p.kqfd, err = syscall.Kqueue()
if p.kqfd == -1 || err != nil {
return nil, err
}
p.event = syscall.Kevent_t{
Ident: uint32(fd),
Filter: syscall.EVFILT_WRITE,
Flags: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT,
Fflags: 0,
Data: 0,
Udata: nil,
}
return p, nil
}
func (p *Poller) Close() error {
return syscall.Close(p.kqfd)
}
func (p *Poller) WaitWrite(deadline time.Time) error {
// setup timeout
var timeout *syscall.Timespec
if !deadline.IsZero() {
d := deadline.Sub(time.Now())
t := syscall.NsecToTimespec(d.Nanoseconds())
timeout = &t
}
// wait on kevent
events := make([]syscall.Kevent_t, 1)
n, err := syscall.Kevent(p.kqfd, []syscall.Kevent_t{p.event}, events, timeout)
if err != nil {
return err
}
if n < 1 {
return errTimeout
}
return nil
}

View File

@ -11,12 +11,18 @@ import (
"fmt"
"os"
"runtime"
"strings"
)
func executable() (string, error) {
switch runtime.GOOS {
case "linux":
return os.Readlink("/proc/self/exe")
const deletedSuffix = " (deleted)"
execpath, err := os.Readlink("/proc/self/exe")
if err != nil {
return execpath, err
}
return strings.TrimSuffix(execpath, deletedSuffix), nil
case "netbsd":
return os.Readlink("/proc/curproc/exe")
case "openbsd", "dragonfly":

View File

@ -7,35 +7,42 @@
package osext
import (
"bytes"
"fmt"
"io"
"os"
oexec "os/exec"
"os/exec"
"path/filepath"
"runtime"
"testing"
)
const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
const (
executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE"
func TestExecPath(t *testing.T) {
executableEnvValueMatch = "match"
executableEnvValueDelete = "delete"
)
func TestExecutableMatch(t *testing.T) {
ep, err := Executable()
if err != nil {
t.Fatalf("ExecPath failed: %v", err)
t.Fatalf("Executable failed: %v", err)
}
// we want fn to be of the form "dir/prog"
// fullpath to be of the form "dir/prog".
dir := filepath.Dir(filepath.Dir(ep))
fn, err := filepath.Rel(dir, ep)
fullpath, err := filepath.Rel(dir, ep)
if err != nil {
t.Fatalf("filepath.Rel: %v", err)
}
cmd := &oexec.Cmd{}
// make child start with a relative program path
cmd.Dir = dir
cmd.Path = fn
// forge argv[0] for child, so that we can verify we could correctly
// get real path of the executable without influenced by argv[0].
cmd.Args = []string{"-", "-test.run=XXXX"}
cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
// Make child start with a relative program path.
// Alter argv[0] for child to verify getting real path without argv[0].
cmd := &exec.Cmd{
Dir: dir,
Path: fullpath,
Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)},
}
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("exec(self) failed: %v", err)
@ -49,6 +56,63 @@ func TestExecPath(t *testing.T) {
}
}
func TestExecutableDelete(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
fpath, err := Executable()
if err != nil {
t.Fatalf("Executable failed: %v", err)
}
r, w := io.Pipe()
stderrBuff := &bytes.Buffer{}
stdoutBuff := &bytes.Buffer{}
cmd := &exec.Cmd{
Path: fpath,
Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)},
Stdin: r,
Stderr: stderrBuff,
Stdout: stdoutBuff,
}
err = cmd.Start()
if err != nil {
t.Fatalf("exec(self) start failed: %v", err)
}
tempPath := fpath + "_copy"
_ = os.Remove(tempPath)
err = copyFile(tempPath, fpath)
if err != nil {
t.Fatalf("copy file failed: %v", err)
}
err = os.Remove(fpath)
if err != nil {
t.Fatalf("remove running test file failed: %v", err)
}
err = os.Rename(tempPath, fpath)
if err != nil {
t.Fatalf("rename copy to previous name failed: %v", err)
}
w.Write([]byte{0})
w.Close()
err = cmd.Wait()
if err != nil {
t.Fatalf("exec wait failed: %v", err)
}
childPath := stderrBuff.String()
if !filepath.IsAbs(childPath) {
t.Fatalf("Child returned %q, want an absolute path", childPath)
}
if !sameFile(childPath, fpath) {
t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath)
}
}
func sameFile(fn1, fn2 string) bool {
fi1, err := os.Stat(fn1)
if err != nil {
@ -60,10 +124,30 @@ func sameFile(fn1, fn2 string) bool {
}
return os.SameFile(fi1, fi2)
}
func copyFile(dest, src string) error {
df, err := os.Create(dest)
if err != nil {
return err
}
defer df.Close()
func init() {
if e := os.Getenv(execPath_EnvVar); e != "" {
// first chdir to another path
sf, err := os.Open(src)
if err != nil {
return err
}
defer sf.Close()
_, err = io.Copy(df, sf)
return err
}
func TestMain(m *testing.M) {
env := os.Getenv(executableEnvVar)
switch env {
case "":
os.Exit(m.Run())
case executableEnvValueMatch:
// First chdir to another path.
dir := "/"
if runtime.GOOS == "windows" {
dir = filepath.VolumeName(".")
@ -74,6 +158,23 @@ func init() {
} else {
fmt.Fprint(os.Stderr, ep)
}
os.Exit(0)
case executableEnvValueDelete:
bb := make([]byte, 1)
var err error
n, err := os.Stdin.Read(bb)
if err != nil {
fmt.Fprint(os.Stderr, "ERROR: ", err)
os.Exit(2)
}
if n != 1 {
fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n)
os.Exit(2)
}
if ep, err := Executable(); err != nil {
fmt.Fprint(os.Stderr, "ERROR: ", err)
} else {
fmt.Fprint(os.Stderr, ep)
}
}
os.Exit(0)
}

View File

@ -4,7 +4,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build go1.3
// +build !go1.2
package leveldb

View File

@ -0,0 +1,30 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build !go1.2
package cache
import (
"math/rand"
"testing"
)
func BenchmarkLRUCache(b *testing.B) {
c := NewCache(NewLRU(10000))
b.SetParallelism(10)
b.RunParallel(func(pb *testing.PB) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for pb.Next() {
key := uint64(r.Intn(1000000))
c.Get(0, key, func() (int, Value) {
return 1, key
}).Release()
}
})
}

View File

@ -8,152 +8,669 @@
package cache
import (
"sync"
"sync/atomic"
"unsafe"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
)
// SetFunc is the function that will be called by Namespace.Get to create
// a cache object, if charge is less than one than the cache object will
// not be registered to cache tree, if value is nil then the cache object
// will not be created.
type SetFunc func() (charge int, value interface{})
// DelFin is the function that will be called as the result of a delete operation.
// Exist == true is indication that the object is exist, and pending == true is
// indication of deletion already happen but haven't done yet (wait for all handles
// to be released). And exist == false means the object doesn't exist.
type DelFin func(exist, pending bool)
// PurgeFin is the function that will be called as the result of a purge operation.
type PurgeFin func(ns, key uint64)
// Cache is a cache tree. A cache instance must be goroutine-safe.
type Cache interface {
// SetCapacity sets cache tree capacity.
SetCapacity(capacity int)
// Capacity returns cache tree capacity.
// Cacher provides interface to implements a caching functionality.
// An implementation must be goroutine-safe.
type Cacher interface {
// Capacity returns cache capacity.
Capacity() int
// Used returns used cache tree capacity.
Used() int
// SetCapacity sets cache capacity.
SetCapacity(capacity int)
// Size returns entire alive cache objects size.
Size() int
// Promote promotes the 'cache node'.
Promote(n *Node)
// NumObjects returns number of alive objects.
NumObjects() int
// Ban evicts the 'cache node' and prevent subsequent 'promote'.
Ban(n *Node)
// GetNamespace gets cache namespace with the given id.
// GetNamespace is never return nil.
GetNamespace(id uint64) Namespace
// Evict evicts the 'cache node'.
Evict(n *Node)
// PurgeNamespace purges cache namespace with the given id from this cache tree.
// Also read Namespace.Purge.
PurgeNamespace(id uint64, fin PurgeFin)
// EvictNS evicts 'cache node' with the given namespace.
EvictNS(ns uint64)
// ZapNamespace detaches cache namespace with the given id from this cache tree.
// Also read Namespace.Zap.
ZapNamespace(id uint64)
// EvictAll evicts all 'cache node'.
EvictAll()
// Purge purges all cache namespace from this cache tree.
// This is behave the same as calling Namespace.Purge method on all cache namespace.
Purge(fin PurgeFin)
// Zap detaches all cache namespace from this cache tree.
// This is behave the same as calling Namespace.Zap method on all cache namespace.
Zap()
// Close closes the 'cache tree'
Close() error
}
// Namespace is a cache namespace. A namespace instance must be goroutine-safe.
type Namespace interface {
// Get gets cache object with the given key.
// If cache object is not found and setf is not nil, Get will atomically creates
// the cache object by calling setf. Otherwise Get will returns nil.
//
// The returned cache handle should be released after use by calling Release
// method.
Get(key uint64, setf SetFunc) Handle
// Value is a 'cacheable object'. It may implements util.Releaser, if
// so the the Release method will be called once object is released.
type Value interface{}
// Delete removes cache object with the given key from cache tree.
// A deleted cache object will be released as soon as all of its handles have
// been released.
// Delete only happen once, subsequent delete will consider cache object doesn't
// exist, even if the cache object ins't released yet.
//
// If not nil, fin will be called if the cache object doesn't exist or when
// finally be released.
//
// Delete returns true if such cache object exist and never been deleted.
Delete(key uint64, fin DelFin) bool
// Purge removes all cache objects within this namespace from cache tree.
// This is the same as doing delete on all cache objects.
//
// If not nil, fin will be called on all cache objects when its finally be
// released.
Purge(fin PurgeFin)
// Zap detaches namespace from cache tree and release all its cache objects.
// A zapped namespace can never be filled again.
// Calling Get on zapped namespace will always return nil.
Zap()
type CacheGetter struct {
Cache *Cache
NS uint64
}
// Handle is a cache handle.
type Handle interface {
// Release releases this cache handle. This method can be safely called mutiple
// times.
Release()
// Value returns value of this cache handle.
// Value will returns nil after this cache handle have be released.
Value() interface{}
func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
return g.Cache.Get(g.NS, key, setFunc)
}
// The hash tables implementation is based on:
// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
const (
DelNotExist = iota
DelExist
DelPendig
mInitialSize = 1 << 4
mOverflowThreshold = 1 << 5
mOverflowGrowThreshold = 1 << 7
)
// Namespace state.
type nsState int
const (
nsEffective nsState = iota
nsZapped
)
// Node state.
type nodeState int
const (
nodeZero nodeState = iota
nodeEffective
nodeEvicted
nodeDeleted
)
// Fake handle.
type fakeHandle struct {
value interface{}
fin func()
once uint32
type mBucket struct {
mu sync.Mutex
node []*Node
frozen bool
}
func (h *fakeHandle) Value() interface{} {
if atomic.LoadUint32(&h.once) == 0 {
return h.value
func (b *mBucket) freeze() []*Node {
b.mu.Lock()
defer b.mu.Unlock()
if !b.frozen {
b.frozen = true
}
return b.node
}
func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
for _, n := range b.node {
if n.hash == hash && n.ns == ns && n.key == key {
atomic.AddInt32(&n.ref, 1)
b.mu.Unlock()
return true, false, n
}
}
// Get only.
if noset {
b.mu.Unlock()
return true, false, nil
}
// Create node.
n = &Node{
r: r,
hash: hash,
ns: ns,
key: key,
ref: 1,
}
// Add node to bucket.
b.node = append(b.node, n)
bLen := len(b.node)
b.mu.Unlock()
// Update counter.
grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
if bLen > mOverflowThreshold {
grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
}
// Grow.
if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) << 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
return true, true, n
}
func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
var (
n *Node
bLen int
)
for i := range b.node {
n = b.node[i]
if n.ns == ns && n.key == key {
if atomic.LoadInt32(&n.ref) == 0 {
deleted = true
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Remove node from bucket.
b.node = append(b.node[:i], b.node[i+1:]...)
bLen = len(b.node)
}
break
}
}
b.mu.Unlock()
if deleted {
// Call OnDel.
for _, f := range n.onDel {
f()
}
// Update counter.
atomic.AddInt32(&r.size, int32(n.size)*-1)
shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
if bLen >= mOverflowThreshold {
atomic.AddInt32(&h.overflow, -1)
}
// Shrink.
if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) >> 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
}
return true, deleted
}
type mNode struct {
buckets []unsafe.Pointer // []*mBucket
mask uint32
pred unsafe.Pointer // *mNode
resizeInProgess int32
overflow int32
growThreshold int32
shrinkThreshold int32
}
func (n *mNode) initBucket(i uint32) *mBucket {
if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
return b
}
p := (*mNode)(atomic.LoadPointer(&n.pred))
if p != nil {
var node []*Node
if n.mask > p.mask {
// Grow.
pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
if pb == nil {
pb = p.initBucket(i & p.mask)
}
m := pb.freeze()
// Split nodes.
for _, x := range m {
if x.hash&n.mask == i {
node = append(node, x)
}
}
} else {
// Shrink.
pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
if pb0 == nil {
pb0 = p.initBucket(i)
}
pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
if pb1 == nil {
pb1 = p.initBucket(i + uint32(len(n.buckets)))
}
m0 := pb0.freeze()
m1 := pb1.freeze()
// Merge nodes.
node = make([]*Node, 0, len(m0)+len(m1))
node = append(node, m0...)
node = append(node, m1...)
}
b := &mBucket{node: node}
if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
if len(node) > mOverflowThreshold {
atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
}
return b
}
}
return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
}
func (n *mNode) initBuckets() {
for i := range n.buckets {
n.initBucket(uint32(i))
}
atomic.StorePointer(&n.pred, nil)
}
// Cache is a 'cache map'.
type Cache struct {
mu sync.RWMutex
mHead unsafe.Pointer // *mNode
nodes int32
size int32
cacher Cacher
closed bool
}
// NewCache creates a new 'cache map'. The cacher is optional and
// may be nil.
func NewCache(cacher Cacher) *Cache {
h := &mNode{
buckets: make([]unsafe.Pointer, mInitialSize),
mask: mInitialSize - 1,
growThreshold: int32(mInitialSize * mOverflowThreshold),
shrinkThreshold: 0,
}
for i := range h.buckets {
h.buckets[i] = unsafe.Pointer(&mBucket{})
}
r := &Cache{
mHead: unsafe.Pointer(h),
cacher: cacher,
}
return r
}
func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
h := (*mNode)(atomic.LoadPointer(&r.mHead))
i := hash & h.mask
b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
if b == nil {
b = h.initBucket(i)
}
return h, b
}
func (r *Cache) delete(n *Node) bool {
for {
h, b := r.getBucket(n.hash)
done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
if done {
return deleted
}
}
return false
}
// Nodes returns number of 'cache node' in the map.
func (r *Cache) Nodes() int {
return int(atomic.LoadInt32(&r.nodes))
}
// Size returns sums of 'cache node' size in the map.
func (r *Cache) Size() int {
return int(atomic.LoadInt32(&r.size))
}
// Capacity returns cache capacity.
func (r *Cache) Capacity() int {
if r.cacher == nil {
return 0
}
return r.cacher.Capacity()
}
// SetCapacity sets cache capacity.
func (r *Cache) SetCapacity(capacity int) {
if r.cacher != nil {
r.cacher.SetCapacity(capacity)
}
}
// Get gets 'cache node' with the given namespace and key.
// If cache node is not found and setFunc is not nil, Get will atomically creates
// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
//
// The returned 'cache handle' should be released after use by calling Release
// method.
func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return nil
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
if done {
if n != nil {
n.mu.Lock()
if n.value == nil {
if setFunc == nil {
n.mu.Unlock()
n.unref()
return nil
}
n.size, n.value = setFunc()
if n.value == nil {
n.size = 0
n.mu.Unlock()
n.unref()
return nil
}
atomic.AddInt32(&r.size, int32(n.size))
}
n.mu.Unlock()
if r.cacher != nil {
r.cacher.Promote(n)
}
return &Handle{unsafe.Pointer(n)}
}
break
}
}
return nil
}
func (h *fakeHandle) Release() {
if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
// Delete removes and ban 'cache node' with the given namespace and key.
// A banned 'cache node' will never inserted into the 'cache tree'. Ban
// only attributed to the particular 'cache node', so when a 'cache node'
// is recreated it will not be banned.
//
// If onDel is not nil, then it will be executed if such 'cache node'
// doesn't exist or once the 'cache node' is released.
//
// Delete return true is such 'cache node' exist.
func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if onDel != nil {
n.mu.Lock()
n.onDel = append(n.onDel, onDel)
n.mu.Unlock()
}
if r.cacher != nil {
r.cacher.Ban(n)
}
n.unref()
return true
}
break
}
}
if onDel != nil {
onDel()
}
return false
}
// Evict evicts 'cache node' with the given namespace and key. This will
// simply call Cacher.Evict.
//
// Evict return true is such 'cache node' exist.
func (r *Cache) Evict(ns, key uint64) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if r.cacher != nil {
r.cacher.Evict(n)
}
n.unref()
return true
}
break
}
}
return false
}
// EvictNS evicts 'cache node' with the given namespace. This will
// simply call Cacher.EvictNS.
func (r *Cache) EvictNS(ns uint64) {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if h.fin != nil {
h.fin()
h.fin = nil
if r.cacher != nil {
r.cacher.EvictNS(ns)
}
}
// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
func (r *Cache) EvictAll() {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if r.cacher != nil {
r.cacher.EvictAll()
}
}
// Close closes the 'cache map' and releases all 'cache node'.
func (r *Cache) Close() error {
r.mu.Lock()
if !r.closed {
r.closed = true
if r.cacher != nil {
if err := r.cacher.Close(); err != nil {
return err
}
}
h := (*mNode)(r.mHead)
h.initBuckets()
for i := range h.buckets {
b := (*mBucket)(h.buckets[i])
for _, n := range b.node {
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Call OnDel.
for _, f := range n.onDel {
f()
}
}
}
}
r.mu.Unlock()
return nil
}
// Node is a 'cache node'.
type Node struct {
r *Cache
hash uint32
ns, key uint64
mu sync.Mutex
size int
value Value
ref int32
onDel []func()
CacheData unsafe.Pointer
}
// NS returns this 'cache node' namespace.
func (n *Node) NS() uint64 {
return n.ns
}
// Key returns this 'cache node' key.
func (n *Node) Key() uint64 {
return n.key
}
// Size returns this 'cache node' size.
func (n *Node) Size() int {
return n.size
}
// Value returns this 'cache node' value.
func (n *Node) Value() Value {
return n.value
}
// Ref returns this 'cache node' ref counter.
func (n *Node) Ref() int32 {
return atomic.LoadInt32(&n.ref)
}
// GetHandle returns an handle for this 'cache node'.
func (n *Node) GetHandle() *Handle {
if atomic.AddInt32(&n.ref, 1) <= 1 {
panic("BUG: Node.GetHandle on zero ref")
}
return &Handle{unsafe.Pointer(n)}
}
func (n *Node) unref() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.delete(n)
}
}
func (n *Node) unrefLocked() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.mu.RLock()
if !n.r.closed {
n.r.delete(n)
}
n.r.mu.RUnlock()
}
}
type Handle struct {
n unsafe.Pointer // *Node
}
func (h *Handle) Value() Value {
n := (*Node)(atomic.LoadPointer(&h.n))
if n != nil {
return n.value
}
return nil
}
func (h *Handle) Release() {
nPtr := atomic.LoadPointer(&h.n)
if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
n := (*Node)(nPtr)
n.unrefLocked()
}
}
func murmur32(ns, key uint64, seed uint32) uint32 {
const (
m = uint32(0x5bd1e995)
r = 24
)
k1 := uint32(ns >> 32)
k2 := uint32(ns)
k3 := uint32(key >> 32)
k4 := uint32(key)
k1 *= m
k1 ^= k1 >> r
k1 *= m
k2 *= m
k2 ^= k2 >> r
k2 *= m
k3 *= m
k3 ^= k3 >> r
k3 *= m
k4 *= m
k4 ^= k4 >> r
k4 *= m
h := seed
h *= m
h ^= k1
h *= m
h ^= k2
h *= m
h ^= k3
h *= m
h ^= k4
h ^= h >> 13
h *= m
h ^= h >> 15
return h
}

View File

@ -13,11 +13,26 @@ import (
"sync/atomic"
"testing"
"time"
"unsafe"
)
type int32o int32
func (o *int32o) acquire() {
if atomic.AddInt32((*int32)(o), 1) != 1 {
panic("BUG: invalid ref")
}
}
func (o *int32o) Release() {
if atomic.AddInt32((*int32)(o), -1) != 0 {
panic("BUG: invalid ref")
}
}
type releaserFunc struct {
fn func()
value interface{}
value Value
}
func (r releaserFunc) Release() {
@ -26,8 +41,8 @@ func (r releaserFunc) Release() {
}
}
func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle {
return ns.Get(key, func() (int, interface{}) {
func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
return c.Get(ns, key, func() (int, Value) {
if relf != nil {
return charge, releaserFunc{relf, value}
} else {
@ -36,7 +51,246 @@ func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) H
})
}
func TestCache_HitMiss(t *testing.T) {
func TestCacheMap(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
nsx := []struct {
nobjects, nhandles, concurrent, repeat int
}{
{10000, 400, 50, 3},
{100000, 1000, 100, 10},
}
var (
objects [][]int32o
handles [][]unsafe.Pointer
)
for _, x := range nsx {
objects = append(objects, make([]int32o, x.nobjects))
handles = append(handles, make([]unsafe.Pointer, x.nhandles))
}
c := NewCache(nil)
wg := new(sync.WaitGroup)
var done int32
for ns, x := range nsx {
for i := 0; i < x.concurrent; i++ {
wg.Add(1)
go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for j := len(objects) * repeat; j >= 0; j-- {
key := uint64(r.Intn(len(objects)))
h := c.Get(uint64(ns), key, func() (int, Value) {
o := &objects[key]
o.acquire()
return 1, o
})
if v := h.Value().(*int32o); v != &objects[key] {
t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
}
if objects[key] != 1 {
t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
}
if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
h.Release()
}
}
}(ns, i, x.repeat, objects[ns], handles[ns])
}
go func(handles []unsafe.Pointer) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for atomic.LoadInt32(&done) == 0 {
i := r.Intn(len(handles))
h := (*Handle)(atomic.LoadPointer(&handles[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
h.Release()
}
time.Sleep(time.Millisecond)
}
}(handles[ns])
}
go func() {
handles := make([]*Handle, 100000)
for atomic.LoadInt32(&done) == 0 {
for i := range handles {
handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
return 1, 1
})
}
for _, h := range handles {
h.Release()
}
}
}()
wg.Wait()
atomic.StoreInt32(&done, 1)
for _, handles0 := range handles {
for i := range handles0 {
h := (*Handle)(atomic.LoadPointer(&handles0[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
h.Release()
}
}
}
for ns, objects0 := range objects {
for i, o := range objects0 {
if o != 0 {
t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
}
}
}
}
func TestCacheMap_NodesAndSize(t *testing.T) {
c := NewCache(nil)
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 2, nil)
set(c, 1, 1, 3, 3, nil)
set(c, 2, 1, 4, 1, nil)
if c.Nodes() != 4 {
t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
}
if c.Size() != 7 {
t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
}
}
func TestLRUCache_Capacity(t *testing.T) {
c := NewCache(NewLRU(10))
if c.Capacity() != 10 {
t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
}
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 2, nil).Release()
set(c, 1, 1, 3, 3, nil).Release()
set(c, 2, 1, 4, 1, nil).Release()
set(c, 2, 2, 5, 1, nil).Release()
set(c, 2, 3, 6, 1, nil).Release()
set(c, 2, 4, 7, 1, nil).Release()
set(c, 2, 5, 8, 1, nil).Release()
if c.Nodes() != 7 {
t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
}
if c.Size() != 10 {
t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
}
c.SetCapacity(9)
if c.Capacity() != 9 {
t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
}
if c.Nodes() != 6 {
t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
}
if c.Size() != 8 {
t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
}
}
func TestCacheMap_NilValue(t *testing.T) {
c := NewCache(NewLRU(10))
h := c.Get(0, 0, func() (size int, value Value) {
return 1, nil
})
if h != nil {
t.Error("cache handle is non-nil")
}
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
}
func TestLRUCache_GetLatency(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
const (
concurrentSet = 30
concurrentGet = 3
duration = 3 * time.Second
delay = 3 * time.Millisecond
maxkey = 100000
)
var (
set, getHit, getAll int32
getMaxLatency, getDuration int64
)
c := NewCache(NewLRU(5000))
wg := &sync.WaitGroup{}
until := time.Now().Add(duration)
for i := 0; i < concurrentSet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for time.Now().Before(until) {
c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
time.Sleep(delay)
atomic.AddInt32(&set, 1)
return 1, 1
}).Release()
}
}(i)
}
for i := 0; i < concurrentGet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for {
mark := time.Now()
if mark.Before(until) {
h := c.Get(0, uint64(r.Intn(maxkey)), nil)
latency := int64(time.Now().Sub(mark))
m := atomic.LoadInt64(&getMaxLatency)
if latency > m {
atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
}
atomic.AddInt64(&getDuration, latency)
if h != nil {
atomic.AddInt32(&getHit, 1)
h.Release()
}
atomic.AddInt32(&getAll, 1)
} else {
break
}
}
}(i)
}
wg.Wait()
getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
if getAvglatency > delay/3 {
t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
}
}
func TestLRUCache_HitMiss(t *testing.T) {
cases := []struct {
key uint64
value string
@ -54,14 +308,13 @@ func TestCache_HitMiss(t *testing.T) {
}
setfin := 0
c := NewLRUCache(1000)
ns := c.GetNamespace(0)
c := NewCache(NewLRU(1000))
for i, x := range cases {
set(ns, x.key, x.value, len(x.value), func() {
set(c, 0, x.key, x.value, len(x.value), func() {
setfin++
}).Release()
for j, y := range cases {
h := ns.Get(y.key, nil)
h := c.Get(0, y.key, nil)
if j <= i {
// should hit
if h == nil {
@ -85,7 +338,7 @@ func TestCache_HitMiss(t *testing.T) {
for i, x := range cases {
finalizerOk := false
ns.Delete(x.key, func(exist, pending bool) {
c.Delete(0, x.key, func() {
finalizerOk = true
})
@ -94,7 +347,7 @@ func TestCache_HitMiss(t *testing.T) {
}
for j, y := range cases {
h := ns.Get(y.key, nil)
h := c.Get(0, y.key, nil)
if j > i {
// should hit
if h == nil {
@ -122,20 +375,19 @@ func TestCache_HitMiss(t *testing.T) {
}
func TestLRUCache_Eviction(t *testing.T) {
c := NewLRUCache(12)
ns := c.GetNamespace(0)
o1 := set(ns, 1, 1, 1, nil)
set(ns, 2, 2, 1, nil).Release()
set(ns, 3, 3, 1, nil).Release()
set(ns, 4, 4, 1, nil).Release()
set(ns, 5, 5, 1, nil).Release()
if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2
c := NewCache(NewLRU(12))
o1 := set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 1, nil).Release()
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
set(c, 0, 5, 5, 1, nil).Release()
if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
h.Release()
}
set(ns, 9, 9, 10, nil).Release() // 5,2,9
set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
for _, key := range []uint64{9, 2, 5, 1} {
h := ns.Get(key, nil)
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
@ -147,7 +399,7 @@ func TestLRUCache_Eviction(t *testing.T) {
}
o1.Release()
for _, key := range []uint64{1, 2, 5} {
h := ns.Get(key, nil)
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
@ -158,7 +410,7 @@ func TestLRUCache_Eviction(t *testing.T) {
}
}
for _, key := range []uint64{3, 4, 9} {
h := ns.Get(key, nil)
h := c.Get(0, key, nil)
if h != nil {
t.Errorf("hit for key '%d'", key)
if x := h.Value().(int); x != int(key) {
@ -169,487 +421,134 @@ func TestLRUCache_Eviction(t *testing.T) {
}
}
func TestLRUCache_SetGet(t *testing.T) {
c := NewLRUCache(13)
ns := c.GetNamespace(0)
for i := 0; i < 200; i++ {
n := uint64(rand.Intn(99999) % 20)
set(ns, n, n, 1, nil).Release()
if h := ns.Get(n, nil); h != nil {
if h.Value() == nil {
t.Errorf("key '%d' contains nil value", n)
func TestLRUCache_Evict(t *testing.T) {
c := NewCache(NewLRU(6))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
set(c, 1, 1, 4, 1, nil).Release()
set(c, 1, 2, 5, 1, nil).Release()
set(c, 2, 1, 6, 1, nil).Release()
set(c, 2, 2, 7, 1, nil).Release()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
h.Release()
} else {
if x := h.Value().(uint64); x != n {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x)
}
t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
}
}
}
if ok := c.Evict(0, 1); !ok {
t.Error("first Cache.Evict on #0.1 return false")
}
if ok := c.Evict(0, 1); ok {
t.Error("second Cache.Evict on #0.1 return true")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
}
c.EvictNS(1)
if h := c.Get(1, 1, nil); h != nil {
t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
}
if h := c.Get(1, 2, nil); h != nil {
t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
}
c.EvictAll()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
}
}
}
}
func TestLRUCache_Delete(t *testing.T) {
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
if ok := c.Delete(0, 1, delFunc); !ok {
t.Error("Cache.Delete on #1 return false")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
}
if ok := c.Delete(0, 1, delFunc); ok {
t.Error("Cache.Delete on #1 return true")
}
h2 := c.Get(0, 2, nil)
if h2 == nil {
t.Error("Cache.Get on #2 return nil")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(1) Cache.Delete on #2 return false")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(2) Cache.Delete on #2 return false")
}
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
c.Get(0, 2, nil).Release()
for key := 2; key <= 4; key++ {
if h := c.Get(0, uint64(key), nil); h != nil {
h.Release()
} else {
t.Errorf("key '%d' doesn't exist", n)
}
}
}
func TestLRUCache_Purge(t *testing.T) {
c := NewLRUCache(3)
ns1 := c.GetNamespace(0)
o1 := set(ns1, 1, 1, 1, nil)
o2 := set(ns1, 2, 2, 1, nil)
ns1.Purge(nil)
set(ns1, 3, 3, 1, nil).Release()
for _, key := range []uint64{1, 2, 3} {
h := ns1.Get(key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
h.Release()
}
}
o1.Release()
o2.Release()
for _, key := range []uint64{1, 2} {
h := ns1.Get(key, nil)
if h != nil {
t.Errorf("hit for key '%d'", key)
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
h.Release()
}
}
}
type testingCacheObjectCounter struct {
created uint
released uint
}
func (c *testingCacheObjectCounter) createOne() {
c.created++
}
func (c *testingCacheObjectCounter) releaseOne() {
c.released++
}
type testingCacheObject struct {
t *testing.T
cnt *testingCacheObjectCounter
ns, key uint64
releaseCalled bool
}
func (x *testingCacheObject) Release() {
if !x.releaseCalled {
x.releaseCalled = true
x.cnt.releaseOne()
} else {
x.t.Errorf("duplicate setfin NS#%d KEY#%d", x.ns, x.key)
}
}
func TestLRUCache_ConcurrentSetGet(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
seed := time.Now().UnixNano()
t.Logf("seed=%d", seed)
const (
N = 2000000
M = 4000
C = 3
)
var set, get uint32
wg := &sync.WaitGroup{}
c := NewLRUCache(M / 4)
for ni := uint64(0); ni < C; ni++ {
r0 := rand.New(rand.NewSource(seed + int64(ni)))
r1 := rand.New(rand.NewSource(seed + int64(ni) + 1))
ns := c.GetNamespace(ni)
wg.Add(2)
go func(ns Namespace, r *rand.Rand) {
for i := 0; i < N; i++ {
x := uint64(r.Int63n(M))
o := ns.Get(x, func() (int, interface{}) {
atomic.AddUint32(&set, 1)
return 1, x
})
if v := o.Value().(uint64); v != x {
t.Errorf("#%d invalid value, got=%d", x, v)
}
o.Release()
}
wg.Done()
}(ns, r0)
go func(ns Namespace, r *rand.Rand) {
for i := 0; i < N; i++ {
x := uint64(r.Int63n(M))
o := ns.Get(x, nil)
if o != nil {
atomic.AddUint32(&get, 1)
if v := o.Value().(uint64); v != x {
t.Errorf("#%d invalid value, got=%d", x, v)
}
o.Release()
}
}
wg.Done()
}(ns, r1)
}
wg.Wait()
t.Logf("set=%d get=%d", set, get)
}
func TestLRUCache_Finalizer(t *testing.T) {
const (
capacity = 100
goroutines = 100
iterations = 10000
keymax = 8000
)
cnt := &testingCacheObjectCounter{}
c := NewLRUCache(capacity)
type instance struct {
seed int64
rnd *rand.Rand
nsid uint64
ns Namespace
effective int
handles []Handle
handlesMap map[uint64]int
delete bool
purge bool
zap bool
wantDel int
delfinCalled int
delfinCalledAll int
delfinCalledEff int
purgefinCalled int
}
instanceGet := func(p *instance, key uint64) {
h := p.ns.Get(key, func() (charge int, value interface{}) {
to := &testingCacheObject{
t: t, cnt: cnt,
ns: p.nsid,
key: key,
}
p.effective++
cnt.createOne()
return 1, releaserFunc{func() {
to.Release()
p.effective--
}, to}
})
p.handles = append(p.handles, h)
p.handlesMap[key] = p.handlesMap[key] + 1
}
instanceRelease := func(p *instance, i int) {
h := p.handles[i]
key := h.Value().(releaserFunc).value.(*testingCacheObject).key
if n := p.handlesMap[key]; n == 0 {
t.Fatal("key ref == 0")
} else if n > 1 {
p.handlesMap[key] = n - 1
} else {
delete(p.handlesMap, key)
}
h.Release()
p.handles = append(p.handles[:i], p.handles[i+1:]...)
p.handles[len(p.handles) : len(p.handles)+1][0] = nil
}
seed := time.Now().UnixNano()
t.Logf("seed=%d", seed)
instances := make([]*instance, goroutines)
for i := range instances {
p := &instance{}
p.handlesMap = make(map[uint64]int)
p.seed = seed + int64(i)
p.rnd = rand.New(rand.NewSource(p.seed))
p.nsid = uint64(i)
p.ns = c.GetNamespace(p.nsid)
p.delete = i%6 == 0
p.purge = i%8 == 0
p.zap = i%12 == 0 || i%3 == 0
instances[i] = p
}
runr := rand.New(rand.NewSource(seed - 1))
run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) {
var (
rx []*instance
rn []int
)
if init == nil {
rx = append([]*instance{}, x...)
rn = make([]int, len(x))
} else {
for _, p := range x {
if init(p) {
rx = append(rx, p)
rn = append(rn, 0)
}
}
}
for len(rx) > 0 {
i := rand.Intn(len(rx))
if fn(rx[i], rn[i]) {
rn[i]++
} else {
rx = append(rx[:i], rx[i+1:]...)
rn = append(rn[:i], rn[i+1:]...)
}
t.Errorf("Cache.Get on #%d return nil", key)
}
}
// Get and release.
run(runr, instances, nil, func(p *instance, i int) bool {
if i < iterations {
if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
instanceGet(p, uint64(p.rnd.Intn(keymax)))
} else {
instanceRelease(p, p.rnd.Intn(len(p.handles)))
}
return true
} else {
return false
}
})
if used, cap := c.Used(), c.Capacity(); used > cap {
t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
h2.Release()
if h := c.Get(0, 2, nil); h != nil {
t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
}
// Check effective objects.
for i, p := range instances {
if int(p.effective) < len(p.handlesMap) {
t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
}
}
if want := int(cnt.created - cnt.released); c.Size() != want {
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
}
// First delete.
run(runr, instances, func(p *instance) bool {
p.wantDel = p.effective
return p.delete
}, func(p *instance, i int) bool {
key := uint64(i)
if key < keymax {
_, wantExist := p.handlesMap[key]
gotExist := p.ns.Delete(key, func(exist, pending bool) {
p.delfinCalledAll++
if exist {
p.delfinCalledEff++
}
})
if !gotExist && wantExist {
t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key)
}
return true
} else {
return false
}
})
// Second delete.
run(runr, instances, func(p *instance) bool {
p.delfinCalled = 0
return p.delete
}, func(p *instance, i int) bool {
key := uint64(i)
if key < keymax {
gotExist := p.ns.Delete(key, func(exist, pending bool) {
if exist && !pending {
t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key)
}
p.delfinCalled++
})
if gotExist {
t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key)
}
return true
} else {
if p.delfinCalled != keymax {
t.Errorf("(2) NS#%d not all delete fin called, diff=%d", p.nsid, keymax-p.delfinCalled)
}
return false
}
})
// Purge.
run(runr, instances, func(p *instance) bool {
return p.purge
}, func(p *instance, i int) bool {
p.ns.Purge(func(ns, key uint64) {
p.purgefinCalled++
})
return false
})
if want := int(cnt.created - cnt.released); c.Size() != want {
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
}
// Release.
run(runr, instances, func(p *instance) bool {
return !p.zap
}, func(p *instance, i int) bool {
if len(p.handles) > 0 {
instanceRelease(p, len(p.handles)-1)
return true
} else {
return false
}
})
if want := int(cnt.created - cnt.released); c.Size() != want {
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
}
// Zap.
run(runr, instances, func(p *instance) bool {
return p.zap
}, func(p *instance, i int) bool {
p.ns.Zap()
p.handles = nil
p.handlesMap = nil
return false
})
if want := int(cnt.created - cnt.released); c.Size() != want {
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
}
if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used {
t.Errorf("Invalid used value, want=%d got=%d", notrel, used)
}
c.Purge(nil)
for _, p := range instances {
if p.delete {
if p.delfinCalledAll != keymax {
t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll)
}
if p.delfinCalledEff != p.wantDel {
t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff)
}
if p.purge && p.purgefinCalled > 0 {
t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled)
}
} else {
if p.purge {
if p.purgefinCalled != p.wantDel {
t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled)
}
}
}
}
if cnt.created != cnt.released {
t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released)
if delFuncCalled != 4 {
t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
}
}
func BenchmarkLRUCache_Set(b *testing.B) {
c := NewLRUCache(0)
ns := c.GetNamespace(0)
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
set(ns, i, "", 1, nil)
}
}
func BenchmarkLRUCache_Get(b *testing.B) {
c := NewLRUCache(0)
ns := c.GetNamespace(0)
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
set(ns, i, "", 1, nil)
}
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
ns.Get(i, nil)
}
}
func BenchmarkLRUCache_Get2(b *testing.B) {
c := NewLRUCache(0)
ns := c.GetNamespace(0)
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
set(ns, i, "", 1, nil)
}
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
ns.Get(i, func() (charge int, value interface{}) {
return 0, nil
})
}
}
func BenchmarkLRUCache_Release(b *testing.B) {
c := NewLRUCache(0)
ns := c.GetNamespace(0)
handles := make([]Handle, b.N)
for i := uint64(0); i < uint64(b.N); i++ {
handles[i] = set(ns, i, "", 1, nil)
}
b.ResetTimer()
for _, h := range handles {
h.Release()
}
}
func BenchmarkLRUCache_SetRelease(b *testing.B) {
capacity := b.N / 100
if capacity <= 0 {
capacity = 10
}
c := NewLRUCache(capacity)
ns := c.GetNamespace(0)
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
set(ns, i, "", 1, nil).Release()
}
}
func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
capacity := b.N / 100
if capacity <= 0 {
capacity = 10
}
c := NewLRUCache(capacity)
ns := c.GetNamespace(0)
b.ResetTimer()
na := b.N / 2
nb := b.N - na
for i := uint64(0); i < uint64(na); i++ {
set(ns, i, "", 1, nil).Release()
}
for i := uint64(0); i < uint64(nb); i++ {
set(ns, i, "", 1, nil).Release()
func TestLRUCache_Close(t *testing.T) {
relFuncCalled := 0
relFunc := func() {
relFuncCalled++
}
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, relFunc).Release()
set(c, 0, 2, 2, 1, relFunc).Release()
h3 := set(c, 0, 3, 3, 1, relFunc)
if h3 == nil {
t.Error("Cache.Get on #3 return nil")
}
if ok := c.Delete(0, 3, delFunc); !ok {
t.Error("Cache.Delete on #3 return false")
}
c.Close()
if relFuncCalled != 3 {
t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
}
if delFuncCalled != 1 {
t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
}
}

View File

@ -0,0 +1,195 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"unsafe"
)
type lruNode struct {
n *Node
h *Handle
ban bool
next, prev *lruNode
}
func (n *lruNode) insert(at *lruNode) {
x := at.next
at.next = n
n.prev = at
n.next = x
x.prev = n
}
func (n *lruNode) remove() {
if n.prev != nil {
n.prev.next = n.next
n.next.prev = n.prev
n.prev = nil
n.next = nil
} else {
panic("BUG: removing removed node")
}
}
type lru struct {
mu sync.Mutex
capacity int
used int
recent lruNode
}
func (r *lru) reset() {
r.recent.next = &r.recent
r.recent.prev = &r.recent
r.used = 0
}
func (r *lru) Capacity() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.capacity
}
func (r *lru) SetCapacity(capacity int) {
var evicted []*lruNode
r.mu.Lock()
r.capacity = capacity
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Promote(n *Node) {
var evicted []*lruNode
r.mu.Lock()
if n.CacheData == nil {
if n.Size() <= r.capacity {
rn := &lruNode{n: n, h: n.GetHandle()}
rn.insert(&r.recent)
n.CacheData = unsafe.Pointer(rn)
r.used += n.Size()
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.insert(&r.recent)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Ban(n *Node) {
r.mu.Lock()
if n.CacheData == nil {
n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.ban = true
r.used -= rn.n.Size()
r.mu.Unlock()
rn.h.Release()
rn.h = nil
return
}
}
r.mu.Unlock()
}
func (r *lru) Evict(n *Node) {
r.mu.Lock()
rn := (*lruNode)(n.CacheData)
if rn == nil || rn.ban {
r.mu.Unlock()
return
}
n.CacheData = nil
r.mu.Unlock()
rn.h.Release()
}
func (r *lru) EvictNS(ns uint64) {
var evicted []*lruNode
r.mu.Lock()
for e := r.recent.prev; e != &r.recent; {
rn := e
e = e.prev
if rn.n.NS() == ns {
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) EvictAll() {
r.mu.Lock()
back := r.recent.prev
for rn := back; rn != &r.recent; rn = rn.prev {
rn.n.CacheData = nil
}
r.reset()
r.mu.Unlock()
for rn := back; rn != &r.recent; rn = rn.prev {
rn.h.Release()
}
}
func (r *lru) Close() error {
return nil
}
// NewLRU create a new LRU-cache.
func NewLRU(capacity int) Cacher {
r := &lru{capacity: capacity}
r.reset()
return r
}

View File

@ -1,622 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"sync/atomic"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
)
// The LLRB implementation were taken from https://github.com/petar/GoLLRB.
// Which contains the following header:
//
// Copyright 2010 Petar Maymounkov. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// lruCache represent a LRU cache state.
type lruCache struct {
mu sync.Mutex
recent lruNode
table map[uint64]*lruNs
capacity int
used, size, alive int
}
// NewLRUCache creates a new initialized LRU cache with the given capacity.
func NewLRUCache(capacity int) Cache {
c := &lruCache{
table: make(map[uint64]*lruNs),
capacity: capacity,
}
c.recent.rNext = &c.recent
c.recent.rPrev = &c.recent
return c
}
func (c *lruCache) Capacity() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.capacity
}
func (c *lruCache) Used() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.used
}
func (c *lruCache) Size() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.size
}
func (c *lruCache) NumObjects() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.alive
}
// SetCapacity set cache capacity.
func (c *lruCache) SetCapacity(capacity int) {
c.mu.Lock()
c.capacity = capacity
c.evict()
c.mu.Unlock()
}
// GetNamespace return namespace object for given id.
func (c *lruCache) GetNamespace(id uint64) Namespace {
c.mu.Lock()
defer c.mu.Unlock()
if ns, ok := c.table[id]; ok {
return ns
}
ns := &lruNs{lru: c, id: id}
c.table[id] = ns
return ns
}
func (c *lruCache) ZapNamespace(id uint64) {
c.mu.Lock()
if ns, exist := c.table[id]; exist {
ns.zapNB()
delete(c.table, id)
}
c.mu.Unlock()
}
func (c *lruCache) PurgeNamespace(id uint64, fin PurgeFin) {
c.mu.Lock()
if ns, exist := c.table[id]; exist {
ns.purgeNB(fin)
}
c.mu.Unlock()
}
// Purge purge entire cache.
func (c *lruCache) Purge(fin PurgeFin) {
c.mu.Lock()
for _, ns := range c.table {
ns.purgeNB(fin)
}
c.mu.Unlock()
}
func (c *lruCache) Zap() {
c.mu.Lock()
for _, ns := range c.table {
ns.zapNB()
}
c.table = make(map[uint64]*lruNs)
c.mu.Unlock()
}
func (c *lruCache) evict() {
top := &c.recent
for n := c.recent.rPrev; c.used > c.capacity && n != top; {
if n.state != nodeEffective {
panic("evicting non effective node")
}
n.state = nodeEvicted
n.rRemove()
n.derefNB()
c.used -= n.charge
n = c.recent.rPrev
}
}
type lruNs struct {
lru *lruCache
id uint64
rbRoot *lruNode
state nsState
}
func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) {
if h == nil {
n = &lruNode{ns: ns, key: key}
return n, n
}
if key < h.key {
hn, n = ns.rbGetOrCreateNode(h.rbLeft, key)
if hn != nil {
h.rbLeft = hn
} else {
return nil, n
}
} else if key > h.key {
hn, n = ns.rbGetOrCreateNode(h.rbRight, key)
if hn != nil {
h.rbRight = hn
} else {
return nil, n
}
} else {
return nil, h
}
if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) {
h = rbRotLeft(h)
}
if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
h = rbRotRight(h)
}
if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
rbFlip(h)
}
return h, n
}
func (ns *lruNs) getOrCreateNode(key uint64) *lruNode {
hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key)
if hn != nil {
ns.rbRoot = hn
ns.rbRoot.rbBlack = true
}
return n
}
func (ns *lruNs) rbGetNode(key uint64) *lruNode {
h := ns.rbRoot
for h != nil {
switch {
case key < h.key:
h = h.rbLeft
case key > h.key:
h = h.rbRight
default:
return h
}
}
return nil
}
func (ns *lruNs) getNode(key uint64) *lruNode {
return ns.rbGetNode(key)
}
func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode {
if h == nil {
return nil
}
if key < h.key {
if h.rbLeft == nil { // key not present. Nothing to delete
return h
}
if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
h = rbMoveLeft(h)
}
h.rbLeft = ns.rbDeleteNode(h.rbLeft, key)
} else {
if rbIsRed(h.rbLeft) {
h = rbRotRight(h)
}
// If @key equals @h.key and no right children at @h
if h.key == key && h.rbRight == nil {
return nil
}
if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) {
h = rbMoveRight(h)
}
// If @key equals @h.key, and (from above) 'h.Right != nil'
if h.key == key {
var x *lruNode
h.rbRight, x = rbDeleteMin(h.rbRight)
if x == nil {
panic("logic")
}
x.rbLeft, h.rbLeft = h.rbLeft, nil
x.rbRight, h.rbRight = h.rbRight, nil
x.rbBlack = h.rbBlack
h = x
} else { // Else, @key is bigger than @h.key
h.rbRight = ns.rbDeleteNode(h.rbRight, key)
}
}
return rbFixup(h)
}
func (ns *lruNs) deleteNode(key uint64) {
ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key)
if ns.rbRoot != nil {
ns.rbRoot.rbBlack = true
}
}
func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool {
if h == nil {
return true
}
if h.key >= pivot {
if !ns.rbIterateNodes(h.rbLeft, pivot, iter) {
return false
}
if !iter(h) {
return false
}
}
return ns.rbIterateNodes(h.rbRight, pivot, iter)
}
func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) {
ns.rbIterateNodes(ns.rbRoot, 0, iter)
}
func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
ns.lru.mu.Lock()
defer ns.lru.mu.Unlock()
if ns.state != nsEffective {
return nil
}
var n *lruNode
if setf == nil {
n = ns.getNode(key)
if n == nil {
return nil
}
} else {
n = ns.getOrCreateNode(key)
}
switch n.state {
case nodeZero:
charge, value := setf()
if value == nil {
ns.deleteNode(key)
return nil
}
if charge < 0 {
charge = 0
}
n.value = value
n.charge = charge
n.state = nodeEvicted
ns.lru.size += charge
ns.lru.alive++
fallthrough
case nodeEvicted:
if n.charge == 0 {
break
}
// Insert to recent list.
n.state = nodeEffective
n.ref++
ns.lru.used += n.charge
ns.lru.evict()
fallthrough
case nodeEffective:
// Bump to front.
n.rRemove()
n.rInsert(&ns.lru.recent)
case nodeDeleted:
// Do nothing.
default:
panic("invalid state")
}
n.ref++
return &lruHandle{node: n}
}
func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
ns.lru.mu.Lock()
defer ns.lru.mu.Unlock()
if ns.state != nsEffective {
if fin != nil {
fin(false, false)
}
return false
}
n := ns.getNode(key)
if n == nil {
if fin != nil {
fin(false, false)
}
return false
}
switch n.state {
case nodeEffective:
ns.lru.used -= n.charge
n.state = nodeDeleted
n.delfin = fin
n.rRemove()
n.derefNB()
case nodeEvicted:
n.state = nodeDeleted
n.delfin = fin
case nodeDeleted:
if fin != nil {
fin(true, true)
}
return false
default:
panic("invalid state")
}
return true
}
func (ns *lruNs) purgeNB(fin PurgeFin) {
if ns.state == nsEffective {
var nodes []*lruNode
ns.iterateNodes(func(n *lruNode) bool {
nodes = append(nodes, n)
return true
})
for _, n := range nodes {
switch n.state {
case nodeEffective:
ns.lru.used -= n.charge
n.state = nodeDeleted
n.purgefin = fin
n.rRemove()
n.derefNB()
case nodeEvicted:
n.state = nodeDeleted
n.purgefin = fin
case nodeDeleted:
default:
panic("invalid state")
}
}
}
}
func (ns *lruNs) Purge(fin PurgeFin) {
ns.lru.mu.Lock()
ns.purgeNB(fin)
ns.lru.mu.Unlock()
}
func (ns *lruNs) zapNB() {
if ns.state == nsEffective {
ns.state = nsZapped
ns.iterateNodes(func(n *lruNode) bool {
if n.state == nodeEffective {
ns.lru.used -= n.charge
n.rRemove()
}
ns.lru.size -= n.charge
n.state = nodeDeleted
n.fin()
return true
})
ns.rbRoot = nil
}
}
func (ns *lruNs) Zap() {
ns.lru.mu.Lock()
ns.zapNB()
delete(ns.lru.table, ns.id)
ns.lru.mu.Unlock()
}
type lruNode struct {
ns *lruNs
rNext, rPrev *lruNode
rbLeft, rbRight *lruNode
rbBlack bool
key uint64
value interface{}
charge int
ref int
state nodeState
delfin DelFin
purgefin PurgeFin
}
func (n *lruNode) rInsert(at *lruNode) {
x := at.rNext
at.rNext = n
n.rPrev = at
n.rNext = x
x.rPrev = n
}
func (n *lruNode) rRemove() bool {
if n.rPrev == nil {
return false
}
n.rPrev.rNext = n.rNext
n.rNext.rPrev = n.rPrev
n.rPrev = nil
n.rNext = nil
return true
}
func (n *lruNode) fin() {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
if n.purgefin != nil {
if n.delfin != nil {
panic("conflicting delete and purge fin")
}
n.purgefin(n.ns.id, n.key)
n.purgefin = nil
} else if n.delfin != nil {
n.delfin(true, false)
n.delfin = nil
}
}
func (n *lruNode) derefNB() {
n.ref--
if n.ref == 0 {
if n.ns.state == nsEffective {
// Remove elemement.
n.ns.deleteNode(n.key)
n.ns.lru.size -= n.charge
n.ns.lru.alive--
n.fin()
}
n.value = nil
} else if n.ref < 0 {
panic("leveldb/cache: lruCache: negative node reference")
}
}
func (n *lruNode) deref() {
n.ns.lru.mu.Lock()
n.derefNB()
n.ns.lru.mu.Unlock()
}
type lruHandle struct {
node *lruNode
once uint32
}
func (h *lruHandle) Value() interface{} {
if atomic.LoadUint32(&h.once) == 0 {
return h.node.value
}
return nil
}
func (h *lruHandle) Release() {
if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
return
}
h.node.deref()
h.node = nil
}
func rbIsRed(h *lruNode) bool {
if h == nil {
return false
}
return !h.rbBlack
}
func rbRotLeft(h *lruNode) *lruNode {
x := h.rbRight
if x.rbBlack {
panic("rotating a black link")
}
h.rbRight = x.rbLeft
x.rbLeft = h
x.rbBlack = h.rbBlack
h.rbBlack = false
return x
}
func rbRotRight(h *lruNode) *lruNode {
x := h.rbLeft
if x.rbBlack {
panic("rotating a black link")
}
h.rbLeft = x.rbRight
x.rbRight = h
x.rbBlack = h.rbBlack
h.rbBlack = false
return x
}
func rbFlip(h *lruNode) {
h.rbBlack = !h.rbBlack
h.rbLeft.rbBlack = !h.rbLeft.rbBlack
h.rbRight.rbBlack = !h.rbRight.rbBlack
}
func rbMoveLeft(h *lruNode) *lruNode {
rbFlip(h)
if rbIsRed(h.rbRight.rbLeft) {
h.rbRight = rbRotRight(h.rbRight)
h = rbRotLeft(h)
rbFlip(h)
}
return h
}
func rbMoveRight(h *lruNode) *lruNode {
rbFlip(h)
if rbIsRed(h.rbLeft.rbLeft) {
h = rbRotRight(h)
rbFlip(h)
}
return h
}
func rbFixup(h *lruNode) *lruNode {
if rbIsRed(h.rbRight) {
h = rbRotLeft(h)
}
if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
h = rbRotRight(h)
}
if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
rbFlip(h)
}
return h
}
func rbDeleteMin(h *lruNode) (hn, n *lruNode) {
if h == nil {
return nil, nil
}
if h.rbLeft == nil {
return nil, h
}
if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
h = rbMoveLeft(h)
}
h.rbLeft, n = rbDeleteMin(h.rbLeft)
return rbFixup(h), n
}

View File

@ -9,14 +9,12 @@ package leveldb
import (
"bytes"
"fmt"
"io"
"math/rand"
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
"io"
"math/rand"
"testing"
)
const ctValSize = 1000
@ -33,8 +31,8 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
return newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
})
}
@ -269,9 +267,9 @@ func TestCorruptDB_TableIndex(t *testing.T) {
func TestCorruptDB_MissingManifest(t *testing.T) {
rnd := rand.New(rand.NewSource(0x0badda7a))
h := newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
Strict: opt.StrictJournalChecksum,
WriteBuffer: 1000 * 60,
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
WriteBuffer: 1000 * 60,
})
h.build(1000)

View File

@ -27,8 +27,7 @@ import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
)
// DB is a LevelDB database.
type DB struct {
type DB struct { // DB is a LevelDB database.
// Need 64-bit alignment.
seq uint64
@ -347,12 +346,14 @@ func recoverTable(s *session, o *opt.Options) error {
return err
}
iter := tr.NewIterator(nil, nil)
iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
if errors.IsCorrupted(err) {
s.logf("table@recovery block corruption @%d %q", file.Num(), err)
tcorruptedBlock++
}
})
if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
itererr.SetErrorCallback(func(err error) {
if errors.IsCorrupted(err) {
s.logf("table@recovery block corruption @%d %q", file.Num(), err)
tcorruptedBlock++
}
})
}
// Scan the table.
for iter.Next() {
@ -823,8 +824,8 @@ func (db *DB) GetProperty(name string) (value string, err error) {
case p == "blockpool":
value = fmt.Sprintf("%v", db.s.tops.bpool)
case p == "cachedblock":
if bc := db.s.o.GetBlockCache(); bc != nil {
value = fmt.Sprintf("%d", bc.Size())
if db.s.tops.bcache != nil {
value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
} else {
value = "<nil>"
}

View File

@ -1271,7 +1271,7 @@ func TestDB_DeletionMarkers2(t *testing.T) {
}
func TestDB_CompactionTableOpenError(t *testing.T) {
h := newDbHarnessWopt(t, &opt.Options{CachedOpenFiles: -1})
h := newDbHarnessWopt(t, &opt.Options{OpenFilesCacheCapacity: -1})
defer h.close()
im := 10
@ -1629,8 +1629,8 @@ func TestDB_ManualCompaction(t *testing.T) {
func TestDB_BloomFilter(t *testing.T) {
h := newDbHarnessWopt(t, &opt.Options{
BlockCache: opt.NoCache,
Filter: filter.NewBloomFilter(10),
DisableBlockCache: true,
Filter: filter.NewBloomFilter(10),
})
defer h.close()
@ -2066,8 +2066,8 @@ func TestDB_GetProperties(t *testing.T) {
func TestDB_GoleveldbIssue72and83(t *testing.T) {
h := newDbHarnessWopt(t, &opt.Options{
WriteBuffer: 1 * opt.MiB,
CachedOpenFiles: 3,
WriteBuffer: 1 * opt.MiB,
OpenFilesCacheCapacity: 3,
})
defer h.close()
@ -2200,7 +2200,7 @@ func TestDB_GoleveldbIssue72and83(t *testing.T) {
func TestDB_TransientError(t *testing.T) {
h := newDbHarnessWopt(t, &opt.Options{
WriteBuffer: 128 * opt.KiB,
CachedOpenFiles: 3,
OpenFilesCacheCapacity: 3,
DisableCompactionBackoff: true,
})
defer h.close()
@ -2410,7 +2410,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
CompactionTableSize: 43 * opt.KiB,
CompactionExpandLimitFactor: 1,
CompactionGPOverlapsFactor: 1,
BlockCache: opt.NoCache,
DisableBlockCache: true,
}
s, err := newSession(stor, o)
if err != nil {

View File

@ -17,14 +17,14 @@ import (
var _ = testutil.Defer(func() {
Describe("Leveldb external", func() {
o := &opt.Options{
BlockCache: opt.NoCache,
BlockRestartInterval: 5,
BlockSize: 80,
Compression: opt.NoCompression,
CachedOpenFiles: -1,
Strict: opt.StrictAll,
WriteBuffer: 1000,
CompactionTableSize: 2000,
DisableBlockCache: true,
BlockRestartInterval: 5,
BlockSize: 80,
Compression: opt.NoCompression,
OpenFilesCacheCapacity: -1,
Strict: opt.StrictAll,
WriteBuffer: 1000,
CompactionTableSize: 2000,
}
Describe("write test", func() {

View File

@ -20,8 +20,9 @@ const (
GiB = MiB * 1024
)
const (
DefaultBlockCacheSize = 8 * MiB
var (
DefaultBlockCacher = LRUCacher
DefaultBlockCacheCapacity = 8 * MiB
DefaultBlockRestartInterval = 16
DefaultBlockSize = 4 * KiB
DefaultCompactionExpandLimitFactor = 25
@ -33,7 +34,8 @@ const (
DefaultCompactionTotalSize = 10 * MiB
DefaultCompactionTotalSizeMultiplier = 10.0
DefaultCompressionType = SnappyCompression
DefaultCachedOpenFiles = 500
DefaultOpenFilesCacher = LRUCacher
DefaultOpenFilesCacheCapacity = 500
DefaultMaxMemCompationLevel = 2
DefaultNumLevel = 7
DefaultWriteBuffer = 4 * MiB
@ -41,22 +43,33 @@ const (
DefaultWriteL0SlowdownTrigger = 8
)
type noCache struct{}
// Cacher is a caching algorithm.
type Cacher interface {
New(capacity int) cache.Cacher
}
func (noCache) SetCapacity(capacity int) {}
func (noCache) Capacity() int { return 0 }
func (noCache) Used() int { return 0 }
func (noCache) Size() int { return 0 }
func (noCache) NumObjects() int { return 0 }
func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
func (noCache) PurgeNamespace(id uint64, fin cache.PurgeFin) {}
func (noCache) ZapNamespace(id uint64) {}
func (noCache) Purge(fin cache.PurgeFin) {}
func (noCache) Zap() {}
type CacherFunc struct {
NewFunc func(capacity int) cache.Cacher
}
var NoCache cache.Cache = noCache{}
func (f *CacherFunc) New(capacity int) cache.Cacher {
if f.NewFunc != nil {
return f.NewFunc(capacity)
}
return nil
}
// Compression is the per-block compression algorithm to use.
func noCacher(int) cache.Cacher { return nil }
var (
// LRUCacher is the LRU-cache algorithm.
LRUCacher = &CacherFunc{cache.NewLRU}
// NoCacher is the value to disable caching algorithm.
NoCacher = &CacherFunc{}
)
// Compression is the 'sorted table' block compression algorithm to use.
type Compression uint
func (c Compression) String() string {
@ -133,16 +146,17 @@ type Options struct {
// The default value is nil
AltFilters []filter.Filter
// BlockCache provides per-block caching for LevelDB. Specify NoCache to
// disable block caching.
// BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching.
// Specify NoCacher to disable caching algorithm.
//
// By default LevelDB will create LRU-cache with capacity of BlockCacheSize.
BlockCache cache.Cache
// The default value is LRUCacher.
BlockCacher Cacher
// BlockCacheSize defines the capacity of the default 'block cache'.
// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
//
// The default value is 8MiB.
BlockCacheSize int
BlockCacheCapacity int
// BlockRestartInterval is the number of keys between restart points for
// delta encoding of keys.
@ -156,13 +170,6 @@ type Options struct {
// The default value is 4KiB.
BlockSize int
// CachedOpenFiles defines number of open files to kept around when not
// in-use, the counting includes still in-use files.
// Set this to negative value to disable caching.
//
// The default value is 500.
CachedOpenFiles int
// CompactionExpandLimitFactor limits compaction size after expanded.
// This will be multiplied by table size limit at compaction target level.
//
@ -237,11 +244,17 @@ type Options struct {
// The default value uses the same ordering as bytes.Compare.
Comparer comparer.Comparer
// Compression defines the per-block compression to use.
// Compression defines the 'sorted table' block compression to use.
//
// The default value (DefaultCompression) uses snappy compression.
Compression Compression
// DisableBlockCache allows disable use of cache.Cache functionality on
// 'sorted table' block.
//
// The default value is false.
DisableBlockCache bool
// DisableCompactionBackoff allows disable compaction retry backoff.
//
// The default value is false.
@ -288,6 +301,18 @@ type Options struct {
// The default is 7.
NumLevel int
// OpenFilesCacher provides cache algorithm for open files caching.
// Specify NoCacher to disable caching algorithm.
//
// The default value is LRUCacher.
OpenFilesCacher Cacher
// OpenFilesCacheCapacity defines the capacity of the open files caching.
// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
//
// The default value is 500.
OpenFilesCacheCapacity int
// Strict defines the DB strict level.
Strict Strict
@ -320,18 +345,22 @@ func (o *Options) GetAltFilters() []filter.Filter {
return o.AltFilters
}
func (o *Options) GetBlockCache() cache.Cache {
if o == nil {
func (o *Options) GetBlockCacher() Cacher {
if o == nil || o.BlockCacher == nil {
return DefaultBlockCacher
} else if o.BlockCacher == NoCacher {
return nil
}
return o.BlockCache
return o.BlockCacher
}
func (o *Options) GetBlockCacheSize() int {
if o == nil || o.BlockCacheSize <= 0 {
return DefaultBlockCacheSize
func (o *Options) GetBlockCacheCapacity() int {
if o == nil || o.BlockCacheCapacity == 0 {
return DefaultBlockCacheCapacity
} else if o.BlockCacheCapacity < 0 {
return 0
}
return o.BlockCacheSize
return o.BlockCacheCapacity
}
func (o *Options) GetBlockRestartInterval() int {
@ -348,15 +377,6 @@ func (o *Options) GetBlockSize() int {
return o.BlockSize
}
func (o *Options) GetCachedOpenFiles() int {
if o == nil || o.CachedOpenFiles == 0 {
return DefaultCachedOpenFiles
} else if o.CachedOpenFiles < 0 {
return 0
}
return o.CachedOpenFiles
}
func (o *Options) GetCompactionExpandLimit(level int) int {
factor := DefaultCompactionExpandLimitFactor
if o != nil && o.CompactionExpandLimitFactor > 0 {
@ -477,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int {
if o != nil {
if o.MaxMemCompationLevel > 0 {
level = o.MaxMemCompationLevel
} else if o.MaxMemCompationLevel == -1 {
} else if o.MaxMemCompationLevel < 0 {
level = 0
}
}
@ -494,6 +514,25 @@ func (o *Options) GetNumLevel() int {
return o.NumLevel
}
func (o *Options) GetOpenFilesCacher() Cacher {
if o == nil || o.OpenFilesCacher == nil {
return DefaultOpenFilesCacher
}
if o.OpenFilesCacher == NoCacher {
return nil
}
return o.OpenFilesCacher
}
func (o *Options) GetOpenFilesCacheCapacity() int {
if o == nil || o.OpenFilesCacheCapacity == 0 {
return DefaultOpenFilesCacheCapacity
} else if o.OpenFilesCacheCapacity < 0 {
return 0
}
return o.OpenFilesCacheCapacity
}
func (o *Options) GetStrict(strict Strict) bool {
if o == nil || o.Strict == 0 {
return DefaultStrict&strict != 0

View File

@ -7,7 +7,6 @@
package leveldb
import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
)
@ -32,13 +31,6 @@ func (s *session) setOptions(o *opt.Options) {
no.AltFilters[i] = &iFilter{filter}
}
}
// Block cache.
switch o.GetBlockCache() {
case nil:
no.BlockCache = cache.NewLRUCache(o.GetBlockCacheSize())
case opt.NoCache:
no.BlockCache = nil
}
// Comparer.
s.icmp = &iComparer{o.GetComparer()}
no.Comparer = s.icmp

View File

@ -73,7 +73,7 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
stCompPtrs: make([]iKey, o.GetNumLevel()),
}
s.setOptions(o)
s.tops = newTableOps(s, s.o.GetCachedOpenFiles())
s.tops = newTableOps(s)
s.setVersion(newVersion(s))
s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
return
@ -82,9 +82,6 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
// Close session.
func (s *session) close() {
s.tops.close()
if bc := s.o.GetBlockCache(); bc != nil {
bc.Purge(nil)
}
if s.manifest != nil {
s.manifest.Close()
}

View File

@ -14,10 +14,8 @@ import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
)
// Logging.
type dropper struct {
s *session
s *session // Logging.
file storage.File
}

View File

@ -286,10 +286,10 @@ func (x *tFilesSortByNum) Less(i, j int) bool {
// Table operations.
type tOps struct {
s *session
cache cache.Cache
cacheNS cache.Namespace
bpool *util.BufferPool
s *session
cache *cache.Cache
bcache *cache.Cache
bpool *util.BufferPool
}
// Creates an empty table and returns table writer.
@ -338,26 +338,28 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
// Opens table. It returns a cache handle, which should
// be released after use.
func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) {
num := f.file.Num()
ch = t.cacheNS.Get(num, func() (charge int, value interface{}) {
ch = t.cache.Get(0, num, func() (size int, value cache.Value) {
var r storage.Reader
r, err = f.file.Open()
if err != nil {
return 0, nil
}
var bcacheNS cache.Namespace
if bc := t.s.o.GetBlockCache(); bc != nil {
bcacheNS = bc.GetNamespace(num)
var bcache *cache.CacheGetter
if t.bcache != nil {
bcache = &cache.CacheGetter{Cache: t.bcache, NS: num}
}
var tr *table.Reader
tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcacheNS, t.bpool, t.s.o.Options)
tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcache, t.bpool, t.s.o.Options)
if err != nil {
r.Close()
return 0, nil
}
return 1, tr
})
if ch == nil && err == nil {
err = ErrClosed
@ -412,16 +414,14 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite
// no one use the the table.
func (t *tOps) remove(f *tFile) {
num := f.file.Num()
t.cacheNS.Delete(num, func(exist, pending bool) {
if !pending {
if err := f.file.Remove(); err != nil {
t.s.logf("table@remove removing @%d %q", num, err)
} else {
t.s.logf("table@remove removed @%d", num)
}
if bc := t.s.o.GetBlockCache(); bc != nil {
bc.ZapNamespace(num)
}
t.cache.Delete(0, num, func() {
if err := f.file.Remove(); err != nil {
t.s.logf("table@remove removing @%d %q", num, err)
} else {
t.s.logf("table@remove removed @%d", num)
}
if t.bcache != nil {
t.bcache.EvictNS(num)
}
})
}
@ -429,18 +429,34 @@ func (t *tOps) remove(f *tFile) {
// Closes the table ops instance. It will close all tables,
// regadless still used or not.
func (t *tOps) close() {
t.cache.Zap()
t.bpool.Close()
t.cache.Close()
if t.bcache != nil {
t.bcache.Close()
}
}
// Creates new initialized table ops instance.
func newTableOps(s *session, cacheCap int) *tOps {
c := cache.NewLRUCache(cacheCap)
func newTableOps(s *session) *tOps {
var (
cacher cache.Cacher
bcache *cache.Cache
)
if s.o.GetOpenFilesCacheCapacity() > 0 {
cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity())
}
if !s.o.DisableBlockCache {
var bcacher cache.Cacher
if s.o.GetBlockCacheCapacity() > 0 {
bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity())
}
bcache = cache.NewCache(bcacher)
}
return &tOps{
s: s,
cache: c,
cacheNS: c.GetNamespace(0),
bpool: util.NewBufferPool(s.o.GetBlockSize() + 5),
s: s,
cache: cache.NewCache(cacher),
bcache: bcache,
bpool: util.NewBufferPool(s.o.GetBlockSize() + 5),
}
}

View File

@ -14,8 +14,6 @@ import (
"strings"
"sync"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
@ -24,6 +22,7 @@ import (
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
)
var (
@ -509,7 +508,7 @@ type Reader struct {
mu sync.RWMutex
fi *storage.FileInfo
reader io.ReaderAt
cache cache.Namespace
cache *cache.CacheGetter
err error
bpool *util.BufferPool
// Options
@ -613,18 +612,22 @@ func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error)
func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) {
if r.cache != nil {
var err error
ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) {
if !fillCache {
return 0, nil
}
var b *block
b, err = r.readBlock(bh, verifyChecksum)
if err != nil {
return 0, nil
}
return cap(b.data), b
})
var (
err error
ch *cache.Handle
)
if fillCache {
ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) {
var b *block
b, err = r.readBlock(bh, verifyChecksum)
if err != nil {
return 0, nil
}
return cap(b.data), b
})
} else {
ch = r.cache.Get(bh.offset, nil)
}
if ch != nil {
b, ok := ch.Value().(*block)
if !ok {
@ -667,18 +670,22 @@ func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) {
func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) {
if r.cache != nil {
var err error
ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) {
if !fillCache {
return 0, nil
}
var b *filterBlock
b, err = r.readFilterBlock(bh)
if err != nil {
return 0, nil
}
return cap(b.data), b
})
var (
err error
ch *cache.Handle
)
if fillCache {
ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) {
var b *filterBlock
b, err = r.readFilterBlock(bh)
if err != nil {
return 0, nil
}
return cap(b.data), b
})
} else {
ch = r.cache.Get(bh.offset, nil)
}
if ch != nil {
b, ok := ch.Value().(*filterBlock)
if !ok {
@ -980,7 +987,7 @@ func (r *Reader) Release() {
// The fi, cache and bpool is optional and can be nil.
//
// The returned table reader instance is goroutine-safe.
func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) (*Reader, error) {
func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache *cache.CacheGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) {
if f == nil {
return nil, errors.New("leveldb/table: nil file")
}

View File

@ -12,12 +12,11 @@ import (
"fmt"
"io"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
)
func sharedPrefixLen(a, b []byte) int {

View File

@ -7,10 +7,15 @@ package snappy
import (
"encoding/binary"
"errors"
"io"
)
// ErrCorrupt reports that the input is invalid.
var ErrCorrupt = errors.New("snappy: corrupt input")
var (
// ErrCorrupt reports that the input is invalid.
ErrCorrupt = errors.New("snappy: corrupt input")
// ErrUnsupported reports that the input isn't supported.
ErrUnsupported = errors.New("snappy: unsupported input")
)
// DecodedLen returns the length of the decoded block.
func DecodedLen(src []byte) (int, error) {
@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) {
}
return dst[:d], nil
}
// NewReader returns a new Reader that decompresses from r, using the framing
// format described at
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
func NewReader(r io.Reader) *Reader {
return &Reader{
r: r,
decoded: make([]byte, maxUncompressedChunkLen),
buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
}
}
// Reader is an io.Reader than can read Snappy-compressed bytes.
type Reader struct {
r io.Reader
err error
decoded []byte
buf []byte
// decoded[i:j] contains decoded bytes that have not yet been passed on.
i, j int
readHeader bool
}
// Reset discards any buffered data, resets all state, and switches the Snappy
// reader to read from r. This permits reusing a Reader rather than allocating
// a new one.
func (r *Reader) Reset(reader io.Reader) {
r.r = reader
r.err = nil
r.i = 0
r.j = 0
r.readHeader = false
}
func (r *Reader) readFull(p []byte) (ok bool) {
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
if r.err == io.ErrUnexpectedEOF {
r.err = ErrCorrupt
}
return false
}
return true
}
// Read satisfies the io.Reader interface.
func (r *Reader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
for {
if r.i < r.j {
n := copy(p, r.decoded[r.i:r.j])
r.i += n
return n, nil
}
if !r.readFull(r.buf[:4]) {
return 0, r.err
}
chunkType := r.buf[0]
if !r.readHeader {
if chunkType != chunkTypeStreamIdentifier {
r.err = ErrCorrupt
return 0, r.err
}
r.readHeader = true
}
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
if chunkLen > len(r.buf) {
r.err = ErrUnsupported
return 0, r.err
}
// The chunk types are specified at
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return 0, r.err
}
buf := r.buf[:chunkLen]
if !r.readFull(buf) {
return 0, r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
buf = buf[checksumSize:]
n, err := DecodedLen(buf)
if err != nil {
r.err = err
return 0, r.err
}
if n > len(r.decoded) {
r.err = ErrCorrupt
return 0, r.err
}
if _, err := Decode(r.decoded, buf); err != nil {
r.err = err
return 0, r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCorrupt
return 0, r.err
}
r.i, r.j = 0, n
continue
case chunkTypeUncompressedData:
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return 0, r.err
}
buf := r.buf[:checksumSize]
if !r.readFull(buf) {
return 0, r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
// Read directly into r.decoded instead of via r.buf.
n := chunkLen - checksumSize
if !r.readFull(r.decoded[:n]) {
return 0, r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCorrupt
return 0, r.err
}
r.i, r.j = 0, n
continue
case chunkTypeStreamIdentifier:
// Section 4.1. Stream identifier (chunk type 0xff).
if chunkLen != len(magicBody) {
r.err = ErrCorrupt
return 0, r.err
}
if !r.readFull(r.buf[:len(magicBody)]) {
return 0, r.err
}
for i := 0; i < len(magicBody); i++ {
if r.buf[i] != magicBody[i] {
r.err = ErrCorrupt
return 0, r.err
}
}
continue
}
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
r.err = ErrUnsupported
return 0, r.err
} else {
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if !r.readFull(r.buf[:chunkLen]) {
return 0, r.err
}
}
}
}

View File

@ -6,6 +6,7 @@ package snappy
import (
"encoding/binary"
"io"
)
// We limit how far copy back-references can go, the same as the C++ code.
@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int {
// This last factor dominates the blowup, so the final estimate is:
return 32 + srcLen + srcLen/6
}
// NewWriter returns a new Writer that compresses to w, using the framing
// format described at
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
func NewWriter(w io.Writer) *Writer {
return &Writer{
w: w,
enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
}
}
// Writer is an io.Writer than can write Snappy-compressed bytes.
type Writer struct {
w io.Writer
err error
enc []byte
buf [checksumSize + chunkHeaderSize]byte
wroteHeader bool
}
// Reset discards the writer's state and switches the Snappy writer to write to
// w. This permits reusing a Writer rather than allocating a new one.
func (w *Writer) Reset(writer io.Writer) {
w.w = writer
w.err = nil
w.wroteHeader = false
}
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (n int, errRet error) {
if w.err != nil {
return 0, w.err
}
if !w.wroteHeader {
copy(w.enc, magicChunk)
if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
w.err = err
return n, err
}
w.wroteHeader = true
}
for len(p) > 0 {
var uncompressed []byte
if len(p) > maxUncompressedChunkLen {
uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
} else {
uncompressed, p = p, nil
}
checksum := crc(uncompressed)
// Compress the buffer, discarding the result if the improvement
// isn't at least 12.5%.
chunkType := uint8(chunkTypeCompressedData)
chunkBody, err := Encode(w.enc, uncompressed)
if err != nil {
w.err = err
return n, err
}
if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
}
chunkLen := 4 + len(chunkBody)
w.buf[0] = chunkType
w.buf[1] = uint8(chunkLen >> 0)
w.buf[2] = uint8(chunkLen >> 8)
w.buf[3] = uint8(chunkLen >> 16)
w.buf[4] = uint8(checksum >> 0)
w.buf[5] = uint8(checksum >> 8)
w.buf[6] = uint8(checksum >> 16)
w.buf[7] = uint8(checksum >> 24)
if _, err = w.w.Write(w.buf[:]); err != nil {
w.err = err
return n, err
}
if _, err = w.w.Write(chunkBody); err != nil {
w.err = err
return n, err
}
n += len(uncompressed)
}
return n, nil
}

View File

@ -8,6 +8,10 @@
// The C++ snappy implementation is at http://code.google.com/p/snappy/
package snappy
import (
"hash/crc32"
)
/*
Each encoded block begins with the varint-encoded length of the decoded data,
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
@ -36,3 +40,29 @@ const (
tagCopy2 = 0x02
tagCopy4 = 0x03
)
const (
checksumSize = 4
chunkHeaderSize = 4
magicChunk = "\xff\x06\x00\x00" + magicBody
magicBody = "sNaPpY"
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
maxUncompressedChunkLen = 65536
)
const (
chunkTypeCompressedData = 0x00
chunkTypeUncompressedData = 0x01
chunkTypePadding = 0xfe
chunkTypeStreamIdentifier = 0xff
)
var crcTable = crc32.MakeTable(crc32.Castagnoli)
// crc implements the checksum specified in section 3 of
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
func crc(b []byte) uint32 {
c := crc32.Update(0, crcTable, b)
return uint32(c>>15|c<<17) + 0xa282ead8
}

View File

@ -18,7 +18,10 @@ import (
"testing"
)
var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
var (
download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
testdata = flag.String("testdata", "testdata", "Directory containing the test data")
)
func roundtrip(b, ebuf, dbuf []byte) error {
e, err := Encode(ebuf, b)
@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) {
}
func TestSmallRand(t *testing.T) {
rand.Seed(27354294)
rng := rand.New(rand.NewSource(27354294))
for n := 1; n < 20000; n += 23 {
b := make([]byte, n)
for i, _ := range b {
b[i] = uint8(rand.Uint32())
for i := range b {
b[i] = uint8(rng.Uint32())
}
if err := roundtrip(b, nil, nil); err != nil {
t.Fatal(err)
@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) {
func TestSmallRegular(t *testing.T) {
for n := 1; n < 20000; n += 23 {
b := make([]byte, n)
for i, _ := range b {
for i := range b {
b[i] = uint8(i%10 + 'a')
}
if err := roundtrip(b, nil, nil); err != nil {
@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) {
}
}
func cmp(a, b []byte) error {
if len(a) != len(b) {
return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
}
for i := range a {
if a[i] != b[i] {
return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
}
}
return nil
}
func TestFramingFormat(t *testing.T) {
// src is comprised of alternating 1e5-sized sequences of random
// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
// because it is larger than maxUncompressedChunkLen (64k).
src := make([]byte, 1e6)
rng := rand.New(rand.NewSource(1))
for i := 0; i < 10; i++ {
if i%2 == 0 {
for j := 0; j < 1e5; j++ {
src[1e5*i+j] = uint8(rng.Intn(256))
}
} else {
for j := 0; j < 1e5; j++ {
src[1e5*i+j] = uint8(i)
}
}
}
buf := new(bytes.Buffer)
if _, err := NewWriter(buf).Write(src); err != nil {
t.Fatalf("Write: encoding: %v", err)
}
dst, err := ioutil.ReadAll(NewReader(buf))
if err != nil {
t.Fatalf("ReadAll: decoding: %v", err)
}
if err := cmp(dst, src); err != nil {
t.Fatal(err)
}
}
func TestReaderReset(t *testing.T) {
gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
buf := new(bytes.Buffer)
if _, err := NewWriter(buf).Write(gold); err != nil {
t.Fatalf("Write: %v", err)
}
encoded, invalid, partial := buf.String(), "invalid", "partial"
r := NewReader(nil)
for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
if s == partial {
r.Reset(strings.NewReader(encoded))
if _, err := r.Read(make([]byte, 101)); err != nil {
t.Errorf("#%d: %v", i, err)
continue
}
continue
}
r.Reset(strings.NewReader(s))
got, err := ioutil.ReadAll(r)
switch s {
case encoded:
if err != nil {
t.Errorf("#%d: %v", i, err)
continue
}
if err := cmp(got, gold); err != nil {
t.Errorf("#%d: %v", i, err)
continue
}
case invalid:
if err == nil {
t.Errorf("#%d: got nil error, want non-nil", i)
continue
}
}
}
}
func TestWriterReset(t *testing.T) {
gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
var gots, wants [][]byte
const n = 20
w, failed := NewWriter(nil), false
for i := 0; i <= n; i++ {
buf := new(bytes.Buffer)
w.Reset(buf)
want := gold[:len(gold)*i/n]
if _, err := w.Write(want); err != nil {
t.Errorf("#%d: Write: %v", i, err)
failed = true
continue
}
got, err := ioutil.ReadAll(NewReader(buf))
if err != nil {
t.Errorf("#%d: ReadAll: %v", i, err)
failed = true
continue
}
gots = append(gots, got)
wants = append(wants, want)
}
if failed {
return
}
for i := range gots {
if err := cmp(gots[i], wants[i]); err != nil {
t.Errorf("#%d: %v", i, err)
}
}
}
func benchDecode(b *testing.B, src []byte) {
encoded, err := Encode(nil, src)
if err != nil {
@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) {
}
}
func readFile(b *testing.B, filename string) []byte {
func readFile(b testing.TB, filename string) []byte {
src, err := ioutil.ReadFile(filename)
if err != nil {
b.Fatalf("failed reading %s: %s", filename, err)
@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
// testFiles' values are copied directly from
// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
// The label field is unused in snappy-go.
var testFiles = []struct {
label string
@ -152,29 +269,36 @@ var testFiles = []struct {
}{
{"html", "html"},
{"urls", "urls.10K"},
{"jpg", "house.jpg"},
{"pdf", "mapreduce-osdi-1.pdf"},
{"jpg", "fireworks.jpeg"},
{"jpg_200", "fireworks.jpeg"},
{"pdf", "paper-100k.pdf"},
{"html4", "html_x_4"},
{"cp", "cp.html"},
{"c", "fields.c"},
{"lsp", "grammar.lsp"},
{"xls", "kennedy.xls"},
{"txt1", "alice29.txt"},
{"txt2", "asyoulik.txt"},
{"txt3", "lcet10.txt"},
{"txt4", "plrabn12.txt"},
{"bin", "ptt5"},
{"sum", "sum"},
{"man", "xargs.1"},
{"pb", "geo.protodata"},
{"gaviota", "kppkn.gtb"},
}
// The test data files are present at this canonical URL.
const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
func downloadTestdata(basename string) (errRet error) {
filename := filepath.Join("testdata", basename)
filename := filepath.Join(*testdata, basename)
if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
return nil
}
if !*download {
return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
}
// Download the official snappy C++ implementation reference test data
// files for benchmarking.
if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to create testdata: %s", err)
}
f, err := os.Create(filename)
if err != nil {
return fmt.Errorf("failed to create %s: %s", filename, err)
@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) {
os.Remove(filename)
}
}()
resp, err := http.Get(baseURL + basename)
url := baseURL + basename
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
return fmt.Errorf("failed to download %s: %s", url, err)
}
defer resp.Body.Close()
if s := resp.StatusCode; s != http.StatusOK {
return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
}
_, err = io.Copy(f, resp.Body)
if err != nil {
return fmt.Errorf("failed to write %s: %s", filename, err)
return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
}
return nil
}
func benchFile(b *testing.B, n int, decode bool) {
filename := filepath.Join("testdata", testFiles[n].filename)
if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
if !*download {
b.Fatal("test data not found; skipping benchmark without the -download flag")
}
// Download the official snappy C++ implementation reference test data
// files for benchmarking.
if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
b.Fatalf("failed to create testdata: %s", err)
}
for _, tf := range testFiles {
if err := downloadTestdata(tf.filename); err != nil {
b.Fatalf("failed to download testdata: %s", err)
}
}
if err := downloadTestdata(testFiles[n].filename); err != nil {
b.Fatalf("failed to download testdata: %s", err)
}
data := readFile(b, filename)
data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
if decode {
benchDecode(b, data)
} else {
@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) }
func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) }
func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) }
func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) }
func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) }
@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) }
func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) }
func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }