From 431d6a5a1407bfbcfb28a798657fa90e68a833a5 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 29 Jul 2015 03:18:24 -0700 Subject: [PATCH] added random-files tool for testing License: MIT Signed-off-by: Juan Batiz-Benet --- Godeps/Godeps.json | 4 + .../github.com/jbenet/go-random-files/LICENSE | 21 ++++ .../jbenet/go-random-files/Makefile | 5 + .../jbenet/go-random-files/README.md | 108 ++++++++++++++++ .../github.com/jbenet/go-random-files/lib.go | 110 ++++++++++++++++ .../go-random-files/random-files/.gitignore | 1 + .../go-random-files/random-files/README.md | 90 +++++++++++++ .../go-random-files/random-files/main.go | 119 ++++++++++++++++++ .../go-random-files/ringreader/ringreader.go | 40 ++++++ .../ringreader/ringreader_test.go | 22 ++++ test/Makefile | 5 + test/sharness/Makefile | 2 +- test/sharness/lib/random-dep.go | 1 + 13 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index d1663a336..4c86e55f8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -186,6 +186,10 @@ "ImportPath": "github.com/jbenet/go-random", "Rev": "cd535bd25356746b9b1e824871dda7da932460e2" }, + { + "ImportPath": "github.com/jbenet/go-random-files", + "Rev": "737479700b40b4b50e914e963ce8d9d44603e3c8" + }, { "ImportPath": "github.com/jbenet/go-reuseport", "Rev": "48959f1fad204b6cf2c0e8d086ef69f03f2de961" diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE b/Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE new file mode 100644 index 000000000..c7386b3c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Juan Batiz-Benet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile b/Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile new file mode 100644 index 000000000..5a72abe1f --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile @@ -0,0 +1,5 @@ +build: + cd random-files && go build + +install: + cd random-files && go install diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md b/Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md new file mode 100644 index 000000000..f2fb4b9b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md @@ -0,0 +1,108 @@ +# go-random-files - create random fs hierarchies + +Useful for testing filesystems. + + +``` +# library +go get -u github.com/jbenet/go-random-files + +# binary +go get -u github.com/jbenet/go-random-files/random-files +``` + +### Library godoc: https://godoc.org/github.com/jbenet/go-random-files + +# random-files - create random fs hierarchies + +See more about the binary at: github.com/jbenet/go-random-files/tree/master/random-files + +Useful for testing filesystems. + +## Install + +``` +go get -u github.com/jbenet/go-random-files/random-files +``` + +## Usage + +```sh +> random-files --help +usage: random-files [options] ... +Write a random filesystem hierarchy to each + +Options: + -alphabet="easy": alphabet for filenames {easy, hard} + -depth=2: fanout depth - how deep the hierarchy goes + -dirs=5: fanout dirs - number of dirs per dir (or max) + -files=10: fanout files - number of files per dir (or max + -filesize=4096: filesize - how big to make each file (or max) + -q=false: quiet output + -random-crypto=false: use cryptographic randomness for files + -random-fanout=false: randomize fanout numbers + -random-size=true: randomize filesize + -seed=0: random seed - 0 for current time +``` + +### Examples + +```sh +> random-files --depth=2 --files=3 foo +foo/h20uo3jrpihb +foo/x6tef1 +foo/jh0c2vdci +foo/fden012m368 +foo/fden012m368/p6n0chy4kg +foo/fden012m368/h92_ +foo/fden012m368/kvjiya98p3 +foo/e_i6hwav1tb +foo/e_i6hwav1tb/oj0-a +foo/e_i6hwav1tb/1-pfgvim +foo/e_i6hwav1tb/s_unf +foo/bgvy8x-_hsm +foo/bgvy8x-_hsm/98zcoz-9ng +foo/bgvy8x-_hsm/j0see3qv +foo/bgvy8x-_hsm/qntuf0r +foo/6zjkw3ejm2awwt +foo/6zjkw3ejm2awwt/iba52dh1lhnewh +foo/6zjkw3ejm2awwt/n1bwcv5zpe +foo/6zjkw3ejm2awwt/o8k89cc +foo/efp_6 +foo/efp_6/qfap2 +foo/efp_6/v_kl_wlefsaa +foo/efp_6/r7sdbph +``` + +It made: + +``` +> tree foo +foo +├── 6zjkw3ejm2awwt +│   ├── iba52dh1lhnewh +│   ├── n1bwcv5zpe +│   └── o8k89cc +├── bgvy8x-_hsm +│   ├── 98zcoz-9ng +│   ├── j0see3qv +│   └── qntuf0r +├── e_i6hwav1tb +│   ├── 1-pfgvim +│   ├── oj0-a +│   └── s_unf +├── efp_6 +│   ├── qfap2 +│   ├── r7sdbph +│   └── v_kl_wlefsaa +├── fden012m368 +│   ├── h92_ +│   ├── kvjiya98p3 +│   └── p6n0chy4kg +├── h20uo3jrpihb +├── jh0c2vdci +└── x6tef1 + +5 directories, 18 files +``` + diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go new file mode 100644 index 000000000..e49ad68e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go @@ -0,0 +1,110 @@ +package randomfiles + +import ( + "fmt" + "io" + "math/rand" + "os" + "path" +) + +type Options struct { + Out io.Writer // output progress + Source io.Reader // randomness source + + FileSize int // the size per file. + Alphabet []rune // for filenames + + FanoutDepth int // how deep the hierarchy goes + FanoutFiles int // how many files per dir + FanoutDirs int // how many dirs per dir + + RandomSeed int64 // use a random seed. if 0, use a random seed + RandomSize bool // randomize file sizes + RandomFanout bool // randomize fanout numbers +} + +func WriteRandomFiles(root string, depth int, opts *Options) error { + + numfiles := opts.FanoutFiles + if opts.RandomFanout { + numfiles = rand.Intn(numfiles) + 1 + } + + for i := 0; i < numfiles; i++ { + if err := WriteRandomFile(root, opts); err != nil { + return err + } + } + + if depth+1 <= opts.FanoutDepth { + numdirs := opts.FanoutDirs + if opts.RandomFanout { + numdirs = rand.Intn(numdirs) + 1 + } + + for i := 0; i < numdirs; i++ { + if err := WriteRandomDir(root, depth+1, opts); err != nil { + return err + } + } + } + + return nil +} + +var FilenameSize = 16 +var RunesEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_") +var RunesHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ") + +func RandomFilename(length int, alphabet []rune) string { + b := make([]rune, length) + for i := range b { + b[i] = alphabet[rand.Intn(len(alphabet))] + } + return string(b) +} + +func WriteRandomFile(root string, opts *Options) error { + filesize := int64(opts.FileSize) + if opts.RandomSize { + filesize = rand.Int63n(filesize) + 1 + } + + n := rand.Intn(FilenameSize-4) + 4 + name := RandomFilename(n, opts.Alphabet) + filepath := path.Join(root, name) + f, err := os.Create(filepath) + if err != nil { + return err + } + + if _, err := io.CopyN(f, opts.Source, filesize); err != nil { + return err + } + + if opts.Out != nil { + fmt.Fprintln(opts.Out, filepath) + } + + return f.Close() +} + +func WriteRandomDir(root string, depth int, opts *Options) error { + if depth > opts.FanoutDepth { + return nil + } + + n := rand.Intn(FilenameSize-4) + 4 + name := RandomFilename(n, opts.Alphabet) + root = path.Join(root, name) + if err := os.MkdirAll(root, 0755); err != nil { + return err + } + + if opts.Out != nil { + fmt.Fprintln(opts.Out, root) + } + + return WriteRandomFiles(root, depth, opts) +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore new file mode 100644 index 000000000..03d7e701f --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore @@ -0,0 +1 @@ +random-files diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md new file mode 100644 index 000000000..acb230e76 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md @@ -0,0 +1,90 @@ +# random-files - create random fs hierarchies + +random-files creates random fs hierarchies. Useful for testing filesystems. + +## Install + +``` +go get -u github.com/jbenet/go-random-files/random-files +``` + +## Usage + +```sh +> random-files --help +usage: random-files [options] ... +Write a random filesystem hierarchy to each + +Options: + -alphabet="easy": alphabet for filenames {easy, hard} + -depth=2: fanout depth - how deep the hierarchy goes + -dirs=5: fanout dirs - number of dirs per dir (or max) + -files=10: fanout files - number of files per dir (or max + -filesize=4096: filesize - how big to make each file (or max) + -q=false: quiet output + -random-crypto=false: use cryptographic randomness for files + -random-fanout=false: randomize fanout numbers + -random-size=true: randomize filesize + -seed=0: random seed - 0 for current time +``` + +## Examples + +```sh +> random-files --depth=2 --files=3 foo +foo/h20uo3jrpihb +foo/x6tef1 +foo/jh0c2vdci +foo/fden012m368 +foo/fden012m368/p6n0chy4kg +foo/fden012m368/h92_ +foo/fden012m368/kvjiya98p3 +foo/e_i6hwav1tb +foo/e_i6hwav1tb/oj0-a +foo/e_i6hwav1tb/1-pfgvim +foo/e_i6hwav1tb/s_unf +foo/bgvy8x-_hsm +foo/bgvy8x-_hsm/98zcoz-9ng +foo/bgvy8x-_hsm/j0see3qv +foo/bgvy8x-_hsm/qntuf0r +foo/6zjkw3ejm2awwt +foo/6zjkw3ejm2awwt/iba52dh1lhnewh +foo/6zjkw3ejm2awwt/n1bwcv5zpe +foo/6zjkw3ejm2awwt/o8k89cc +foo/efp_6 +foo/efp_6/qfap2 +foo/efp_6/v_kl_wlefsaa +foo/efp_6/r7sdbph +``` + +It made: + +``` +> tree foo +foo +├── 6zjkw3ejm2awwt +│   ├── iba52dh1lhnewh +│   ├── n1bwcv5zpe +│   └── o8k89cc +├── bgvy8x-_hsm +│   ├── 98zcoz-9ng +│   ├── j0see3qv +│   └── qntuf0r +├── e_i6hwav1tb +│   ├── 1-pfgvim +│   ├── oj0-a +│   └── s_unf +├── efp_6 +│   ├── qfap2 +│   ├── r7sdbph +│   └── v_kl_wlefsaa +├── fden012m368 +│   ├── h92_ +│   ├── kvjiya98p3 +│   └── p6n0chy4kg +├── h20uo3jrpihb +├── jh0c2vdci +└── x6tef1 + +5 directories, 18 files +``` diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go new file mode 100644 index 000000000..b466916a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go @@ -0,0 +1,119 @@ +package main + +import ( + crand "crypto/rand" + "errors" + "flag" + "fmt" + "math/rand" + "os" + "time" + + randomfiles "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files" + ringreader "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader" +) + +var usage = `usage: %s [options] ... +Write a random filesystem hierarchy to each + +Options: +` + +// flags +var opts randomfiles.Options +var quiet bool +var alphabet string +var paths []string +var cryptorand bool + +func init() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, usage, os.Args[0]) + flag.PrintDefaults() + } + + flag.BoolVar(&quiet, "q", false, "quiet output") + flag.BoolVar(&cryptorand, "random-crypto", false, "use cryptographic randomness for files") + flag.StringVar(&alphabet, "alphabet", "easy", "alphabet for filenames {easy, hard}") + flag.IntVar(&opts.FileSize, "filesize", 4096, "filesize - how big to make each file (or max)") + + flag.IntVar(&opts.FanoutDepth, "depth", 2, "fanout depth - how deep the hierarchy goes") + flag.IntVar(&opts.FanoutDirs, "dirs", 5, "fanout dirs - number of dirs per dir (or max)") + flag.IntVar(&opts.FanoutFiles, "files", 10, "fanout files - number of files per dir (or max") + + flag.Int64Var(&opts.RandomSeed, "seed", 0, "random seed - 0 for current time") + flag.BoolVar(&opts.RandomFanout, "random-fanout", false, "randomize fanout numbers") + flag.BoolVar(&opts.RandomSize, "random-size", true, "randomize filesize") +} + +func parseArgs() error { + flag.Parse() + + switch alphabet { + case "easy": + opts.Alphabet = randomfiles.RunesEasy + case "hard": + opts.Alphabet = randomfiles.RunesHard + default: + return errors.New("alphabet must be one of: easy, hard") + } + + paths = flag.Args() + if len(paths) < 1 { + flag.Usage() + os.Exit(0) + } + + if !quiet { + opts.Out = os.Stdout + } + + switch opts.RandomSeed { + case 0: + rand.Seed(time.Now().UnixNano()) + default: + rand.Seed(opts.RandomSeed) + } + + // prepare randomn source. + if cryptorand { + opts.Source = crand.Reader + } else { + // if not crypto, we don't need a lot of random + // data. we just need to sample from a sequence. + s := 16777216 // 16MB + r, err := ringreader.NewReader(s) + if err != nil { + return err + } + opts.Source = r + } + + return nil +} + +func run() error { + if err := parseArgs(); err != nil { + return err + } + + for _, root := range paths { + if err := os.MkdirAll(root, 0755); err != nil { + return err + } + + err := randomfiles.WriteRandomFiles(root, 1, &opts) + if err != nil { + return err + } + } + + return nil +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go new file mode 100644 index 000000000..c40dbb424 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go @@ -0,0 +1,40 @@ +package ringreader + +import ( + "bytes" + "fmt" + "math/rand" + + random "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" +) + +type Reader struct { + Buf []byte +} + +func NewReader(bufsize int) (*Reader, error) { + buf := bytes.NewBuffer(nil) + err := random.WritePseudoRandomBytes(int64(bufsize), buf, rand.Int63()) + return &Reader{Buf: buf.Bytes()}, err +} + +func (r *Reader) Read(buf []byte) (n int, err error) { + ibufl := len(r.Buf) + left := len(buf) + copied := 0 + + for copied < left { + pos1 := rand.Intn(len(r.Buf)) + pos2 := pos1 + left + if pos2 > ibufl { + pos2 = ibufl + } + copied += copy(buf[copied:], r.Buf[pos1:pos2]) + } + + if copied != left { + err := fmt.Errorf("copied a different ammount: %d != %d", copied, left) + panic(err.Error()) + } + return copied, nil +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go new file mode 100644 index 000000000..791aa070e --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go @@ -0,0 +1,22 @@ +package ringreader + +import ( + "testing" +) + +func TestRingReader(t *testing.T) { + r, _ := NewReader(256) + t.Log("buffer:", r.Buf) + + for i := 1; i < 1048576; i = i * 2 { + buf := make([]byte, i) + n, err := r.Read(buf) + if err != nil { + t.Error(err) + } + if n != len(buf) { + t.Error("did not read %d bytes", n) + } + t.Log("read:", buf) + } +} diff --git a/test/Makefile b/test/Makefile index cc5e1e933..ccc33e181 100644 --- a/test/Makefile +++ b/test/Makefile @@ -3,6 +3,7 @@ BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep IPFS_ROOT = ../ IPFS_CMD = ../cmd/ipfs RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random +RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash IPTB_SRC = ../Godeps/_workspace/src/github.com/whyrusleeping/iptb POLLENDPOINT_SRC= ../thirdparty/pollEndpoint @@ -26,6 +27,10 @@ bin/random: $(call find_go_files, $(RANDOM_SRC)) IPFS-BUILD-OPTIONS @echo "*** installing $@ ***" go build $(GOFLAGS) -o bin/random $(RANDOM_SRC)/random +bin/random-files: + @echo "*** installing $@ ***" + go build $(GOFLAGS) -o bin/random-files $(RANDOM_FILES_SRC)/random-files + bin/multihash: $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS @echo "*** installing $@ ***" go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash diff --git a/test/sharness/Makefile b/test/sharness/Makefile index fd3869428..20431db54 100644 --- a/test/sharness/Makefile +++ b/test/sharness/Makefile @@ -8,7 +8,7 @@ T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \ - bin/iptb bin/go-sleep + bin/iptb bin/go-sleep bin/random-files SHARNESS = lib/sharness/sharness.sh IPFS_ROOT = ../.. diff --git a/test/sharness/lib/random-dep.go b/test/sharness/lib/random-dep.go index 643f08ea8..94b758d7b 100644 --- a/test/sharness/lib/random-dep.go +++ b/test/sharness/lib/random-dep.go @@ -5,4 +5,5 @@ package randomdep import ( _ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" + _ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files" )