From c613fbec3870f7cf97fc271dcbf69ccf34c1c0b5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 5 Feb 2018 20:28:27 +0100 Subject: [PATCH] WIP: Extract: importers/chunk module as go-ipfs-chunker License: MIT Signed-off-by: Hector Sanjuan --- core/corehttp/gateway_handler.go | 2 +- core/coreunix/add.go | 2 +- core/coreunix/metadata_test.go | 2 +- fuse/readonly/ipfs_test.go | 2 +- importer/balanced/balanced_test.go | 2 +- importer/chunk/parse.go | 79 ------------------- importer/chunk/rabin.go | 54 ------------- importer/chunk/rabin_test.go | 81 ------------------- importer/chunk/splitting.go | 105 ------------------------- importer/chunk/splitting_test.go | 120 ----------------------------- importer/helpers/dagbuilder.go | 2 +- importer/importer.go | 2 +- importer/importer_test.go | 2 +- importer/trickle/trickle_test.go | 2 +- merkledag/merkledag_test.go | 2 +- mfs/file.go | 2 +- mfs/mfs_test.go | 2 +- package.json | 5 ++ tar/format.go | 2 +- unixfs/mod/dagmodifier.go | 2 +- unixfs/test/utils.go | 2 +- 21 files changed, 20 insertions(+), 454 deletions(-) delete mode 100644 importer/chunk/parse.go delete mode 100644 importer/chunk/rabin.go delete mode 100644 importer/chunk/rabin_test.go delete mode 100644 importer/chunk/splitting.go delete mode 100644 importer/chunk/splitting_test.go diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 5048821ea..41e532638 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -16,12 +16,12 @@ import ( coreapi "github.com/ipfs/go-ipfs/core/coreapi" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" dagutils "github.com/ipfs/go-ipfs/merkledag/utils" path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing" diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 2a19b82f4..9aafd2034 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -15,7 +15,6 @@ import ( core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/exchange/offline" balanced "github.com/ipfs/go-ipfs/importer/balanced" - "github.com/ipfs/go-ipfs/importer/chunk" ihelper "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" dag "github.com/ipfs/go-ipfs/merkledag" @@ -26,6 +25,7 @@ import ( ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" syncds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore/sync" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files" diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go index cee404585..1a9034f82 100644 --- a/core/coreunix/metadata_test.go +++ b/core/coreunix/metadata_test.go @@ -11,10 +11,10 @@ import ( core "github.com/ipfs/go-ipfs/core" offline "github.com/ipfs/go-ipfs/exchange/offline" importer "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" merkledag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index b423034af..aa6d4d3e5 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -17,9 +17,9 @@ import ( coreunix "github.com/ipfs/go-ipfs/core/coreunix" coremock "github.com/ipfs/go-ipfs/core/mock" importer "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" ci "gx/ipfs/QmVvkK7s5imCiq3JVbL3pGfnhcCnf3LrFJPF4GE2sAoGZf/go-testutil/ci" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index c787cce18..04c94c5a5 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -9,11 +9,11 @@ import ( mrand "math/rand" "testing" - chunk "github.com/ipfs/go-ipfs/importer/chunk" h "github.com/ipfs/go-ipfs/importer/helpers" dag "github.com/ipfs/go-ipfs/merkledag" mdtest "github.com/ipfs/go-ipfs/merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/importer/chunk/parse.go b/importer/chunk/parse.go deleted file mode 100644 index 7d511c217..000000000 --- a/importer/chunk/parse.go +++ /dev/null @@ -1,79 +0,0 @@ -package chunk - -import ( - "errors" - "fmt" - "io" - "strconv" - "strings" -) - -// FromString returns a Splitter depending on the given string: -// it supports "default" (""), "size-{size}", "rabin", "rabin-{blocksize}" and -// "rabin-{min}-{avg}-{max}". -func FromString(r io.Reader, chunker string) (Splitter, error) { - switch { - case chunker == "" || chunker == "default": - return DefaultSplitter(r), nil - - case strings.HasPrefix(chunker, "size-"): - sizeStr := strings.Split(chunker, "-")[1] - size, err := strconv.Atoi(sizeStr) - if err != nil { - return nil, err - } - return NewSizeSplitter(r, int64(size)), nil - - case strings.HasPrefix(chunker, "rabin"): - return parseRabinString(r, chunker) - - default: - return nil, fmt.Errorf("unrecognized chunker option: %s", chunker) - } -} - -func parseRabinString(r io.Reader, chunker string) (Splitter, error) { - parts := strings.Split(chunker, "-") - switch len(parts) { - case 1: - return NewRabin(r, uint64(DefaultBlockSize)), nil - case 2: - size, err := strconv.Atoi(parts[1]) - if err != nil { - return nil, err - } - return NewRabin(r, uint64(size)), nil - case 4: - sub := strings.Split(parts[1], ":") - if len(sub) > 1 && sub[0] != "min" { - return nil, errors.New("first label must be min") - } - min, err := strconv.Atoi(sub[len(sub)-1]) - if err != nil { - return nil, err - } - - sub = strings.Split(parts[2], ":") - if len(sub) > 1 && sub[0] != "avg" { - log.Error("sub == ", sub) - return nil, errors.New("second label must be avg") - } - avg, err := strconv.Atoi(sub[len(sub)-1]) - if err != nil { - return nil, err - } - - sub = strings.Split(parts[3], ":") - if len(sub) > 1 && sub[0] != "max" { - return nil, errors.New("final label must be max") - } - max, err := strconv.Atoi(sub[len(sub)-1]) - if err != nil { - return nil, err - } - - return NewRabinMinMax(r, uint64(min), uint64(avg), uint64(max)), nil - default: - return nil, errors.New("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'") - } -} diff --git a/importer/chunk/rabin.go b/importer/chunk/rabin.go deleted file mode 100644 index c3d1ebdba..000000000 --- a/importer/chunk/rabin.go +++ /dev/null @@ -1,54 +0,0 @@ -package chunk - -import ( - "hash/fnv" - "io" - - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/whyrusleeping/chunker" -) - -// IpfsRabinPoly is the irreducible polynomial of degree 53 used by for Rabin. -var IpfsRabinPoly = chunker.Pol(17437180132763653) - -// Rabin implements the Splitter interface and splits content with Rabin -// fingerprints. -type Rabin struct { - r *chunker.Chunker - reader io.Reader -} - -// NewRabin creates a new Rabin splitter with the given -// average block size. -func NewRabin(r io.Reader, avgBlkSize uint64) *Rabin { - min := avgBlkSize / 3 - max := avgBlkSize + (avgBlkSize / 2) - - return NewRabinMinMax(r, min, avgBlkSize, max) -} - -// NewRabinMinMax returns a new Rabin splitter which uses -// the given min, average and max block sizes. -func NewRabinMinMax(r io.Reader, min, avg, max uint64) *Rabin { - h := fnv.New32a() - ch := chunker.New(r, IpfsRabinPoly, h, avg, min, max) - - return &Rabin{ - r: ch, - reader: r, - } -} - -// NextBytes reads the next bytes from the reader and returns a slice. -func (r *Rabin) NextBytes() ([]byte, error) { - ch, err := r.r.Next() - if err != nil { - return nil, err - } - - return ch.Data, nil -} - -// Reader returns the io.Reader associated to this Splitter. -func (r *Rabin) Reader() io.Reader { - return r.reader -} diff --git a/importer/chunk/rabin_test.go b/importer/chunk/rabin_test.go deleted file mode 100644 index 2f68f01c4..000000000 --- a/importer/chunk/rabin_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package chunk - -import ( - "bytes" - "fmt" - "io" - "testing" - - util "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" -) - -func TestRabinChunking(t *testing.T) { - data := make([]byte, 1024*1024*16) - util.NewTimeSeededRand().Read(data) - - r := NewRabin(bytes.NewReader(data), 1024*256) - - var chunks [][]byte - - for { - chunk, err := r.NextBytes() - if err != nil { - if err == io.EOF { - break - } - t.Fatal(err) - } - - chunks = append(chunks, chunk) - } - - fmt.Printf("average block size: %d\n", len(data)/len(chunks)) - - unchunked := bytes.Join(chunks, nil) - if !bytes.Equal(unchunked, data) { - fmt.Printf("%d %d\n", len(unchunked), len(data)) - t.Fatal("data was chunked incorrectly") - } -} - -func chunkData(t *testing.T, data []byte) map[string]blocks.Block { - r := NewRabin(bytes.NewReader(data), 1024*256) - - blkmap := make(map[string]blocks.Block) - - for { - blk, err := r.NextBytes() - if err != nil { - if err == io.EOF { - break - } - t.Fatal(err) - } - - b := blocks.NewBlock(blk) - blkmap[b.Cid().KeyString()] = b - } - - return blkmap -} - -func TestRabinChunkReuse(t *testing.T) { - data := make([]byte, 1024*1024*16) - util.NewTimeSeededRand().Read(data) - - ch1 := chunkData(t, data[1000:]) - ch2 := chunkData(t, data) - - var extra int - for k := range ch2 { - _, ok := ch1[k] - if !ok { - extra++ - } - } - - if extra > 2 { - t.Log("too many spare chunks made") - } -} diff --git a/importer/chunk/splitting.go b/importer/chunk/splitting.go deleted file mode 100644 index 5be27625b..000000000 --- a/importer/chunk/splitting.go +++ /dev/null @@ -1,105 +0,0 @@ -// Package chunk implements streaming block splitters. -// Splitters read data from a reader and provide byte slices (chunks) -// The size and contents of these slices depend on the splitting method -// used. -package chunk - -import ( - "io" - - logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" - mpool "gx/ipfs/QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG/go-msgio/mpool" -) - -var log = logging.Logger("chunk") - -// DefaultBlockSize is the chunk size that splitters produce (or aim to). -var DefaultBlockSize int64 = 1024 * 256 - -// A Splitter reads bytes from a Reader and creates "chunks" (byte slices) -// that can be used to build DAG nodes. -type Splitter interface { - Reader() io.Reader - NextBytes() ([]byte, error) -} - -// SplitterGen is a splitter generator, given a reader. -type SplitterGen func(r io.Reader) Splitter - -// DefaultSplitter returns a SizeSplitter with the DefaultBlockSize. -func DefaultSplitter(r io.Reader) Splitter { - return NewSizeSplitter(r, DefaultBlockSize) -} - -// SizeSplitterGen returns a SplitterGen function which will create -// a splitter with the given size when called. -func SizeSplitterGen(size int64) SplitterGen { - return func(r io.Reader) Splitter { - return NewSizeSplitter(r, size) - } -} - -// Chan returns a channel that receives each of the chunks produced -// by a splitter, along with another one for errors. -func Chan(s Splitter) (<-chan []byte, <-chan error) { - out := make(chan []byte) - errs := make(chan error, 1) - go func() { - defer close(out) - defer close(errs) - - // all-chunks loop (keep creating chunks) - for { - b, err := s.NextBytes() - if err != nil { - errs <- err - return - } - - out <- b - } - }() - return out, errs -} - -type sizeSplitterv2 struct { - r io.Reader - size uint32 - err error -} - -// NewSizeSplitter returns a new size-based Splitter with the given block size. -func NewSizeSplitter(r io.Reader, size int64) Splitter { - return &sizeSplitterv2{ - r: r, - size: uint32(size), - } -} - -// NextBytes produces a new chunk. -func (ss *sizeSplitterv2) NextBytes() ([]byte, error) { - if ss.err != nil { - return nil, ss.err - } - - full := mpool.ByteSlicePool.Get(ss.size).([]byte)[:ss.size] - n, err := io.ReadFull(ss.r, full) - switch err { - case io.ErrUnexpectedEOF: - ss.err = io.EOF - small := make([]byte, n) - copy(small, full) - mpool.ByteSlicePool.Put(ss.size, full) - return small, nil - case nil: - return full, nil - default: - mpool.ByteSlicePool.Put(ss.size, full) - return nil, err - } -} - -// Reader returns the io.Reader associated to this Splitter. -func (ss *sizeSplitterv2) Reader() io.Reader { - return ss.r -} diff --git a/importer/chunk/splitting_test.go b/importer/chunk/splitting_test.go deleted file mode 100644 index c5ef621e0..000000000 --- a/importer/chunk/splitting_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package chunk - -import ( - "bytes" - "io" - "testing" - - u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" -) - -func randBuf(t *testing.T, size int) []byte { - buf := make([]byte, size) - if _, err := u.NewTimeSeededRand().Read(buf); err != nil { - t.Fatal("failed to read enough randomness") - } - return buf -} - -func copyBuf(buf []byte) []byte { - cpy := make([]byte, len(buf)) - copy(cpy, buf) - return cpy -} - -func TestSizeSplitterOverAllocate(t *testing.T) { - max := 1000 - r := bytes.NewReader(randBuf(t, max)) - chunksize := int64(1024 * 256) - splitter := NewSizeSplitter(r, chunksize) - chunk, err := splitter.NextBytes() - if err != nil { - t.Fatal(err) - } - if cap(chunk) > len(chunk) { - t.Fatal("chunk capacity too large") - } -} - -func TestSizeSplitterIsDeterministic(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - test := func() { - bufR := randBuf(t, 10000000) // crank this up to satisfy yourself. - bufA := copyBuf(bufR) - bufB := copyBuf(bufR) - - chunksA, _ := Chan(DefaultSplitter(bytes.NewReader(bufA))) - chunksB, _ := Chan(DefaultSplitter(bytes.NewReader(bufB))) - - for n := 0; ; n++ { - a, moreA := <-chunksA - b, moreB := <-chunksB - - if !moreA { - if moreB { - t.Fatal("A ended, B didnt.") - } - return - } - - if !bytes.Equal(a, b) { - t.Fatalf("chunk %d not equal", n) - } - } - } - - for run := 0; run < 1; run++ { // crank this up to satisfy yourself. - test() - } -} - -func TestSizeSplitterFillsChunks(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - max := 10000000 - b := randBuf(t, max) - r := &clipReader{r: bytes.NewReader(b), size: 4000} - chunksize := int64(1024 * 256) - c, _ := Chan(NewSizeSplitter(r, chunksize)) - - sofar := 0 - whole := make([]byte, max) - for chunk := range c { - - bc := b[sofar : sofar+len(chunk)] - if !bytes.Equal(bc, chunk) { - t.Fatalf("chunk not correct: (sofar: %d) %d != %d, %v != %v", sofar, len(bc), len(chunk), bc[:100], chunk[:100]) - } - - copy(whole[sofar:], chunk) - - sofar += len(chunk) - if sofar != max && len(chunk) < int(chunksize) { - t.Fatal("sizesplitter split at a smaller size") - } - } - - if !bytes.Equal(b, whole) { - t.Fatal("splitter did not split right") - } -} - -type clipReader struct { - size int - r io.Reader -} - -func (s *clipReader) Read(buf []byte) (int, error) { - - // clip the incoming buffer to produce smaller chunks - if len(buf) > s.size { - buf = buf[:s.size] - } - - return s.r.Read(buf) -} diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index ad36bfcef..0c456bf9d 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -5,10 +5,10 @@ import ( "io" "os" - "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/importer/importer.go b/importer/importer.go index 7e53eb776..b5b631658 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -6,11 +6,11 @@ import ( "fmt" "os" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" bal "github.com/ipfs/go-ipfs/importer/balanced" - "github.com/ipfs/go-ipfs/importer/chunk" h "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" ) diff --git a/importer/importer_test.go b/importer/importer_test.go index c9a7a98ab..4b18c494a 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -7,9 +7,9 @@ import ( "io/ioutil" "testing" - chunk "github.com/ipfs/go-ipfs/importer/chunk" mdtest "github.com/ipfs/go-ipfs/merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index 618bf69eb..0bc5d692f 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -9,12 +9,12 @@ import ( mrand "math/rand" "testing" - chunk "github.com/ipfs/go-ipfs/importer/chunk" h "github.com/ipfs/go-ipfs/importer/helpers" merkledag "github.com/ipfs/go-ipfs/merkledag" mdtest "github.com/ipfs/go-ipfs/merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index e0b9d8b80..02cf56ce0 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -17,13 +17,13 @@ import ( bstest "github.com/ipfs/go-ipfs/blockservice/test" offline "github.com/ipfs/go-ipfs/exchange/offline" imp "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" . "github.com/ipfs/go-ipfs/merkledag" mdpb "github.com/ipfs/go-ipfs/merkledag/pb" dstest "github.com/ipfs/go-ipfs/merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" diff --git a/mfs/file.go b/mfs/file.go index 496b05d8e..fc7a9ef32 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -5,10 +5,10 @@ import ( "fmt" "sync" - chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" mod "github.com/ipfs/go-ipfs/unixfs/mod" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 5db5d4987..187481077 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -18,11 +18,11 @@ import ( bserv "github.com/ipfs/go-ipfs/blockservice" offline "github.com/ipfs/go-ipfs/exchange/offline" importer "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" diff --git a/package.json b/package.json index fbdccda13..293d8a17c 100644 --- a/package.json +++ b/package.json @@ -521,6 +521,11 @@ "hash": "Qmb3jLEFAQrqdVgWUajqEyuuDoavkSq1XQXz6tWdFWF995", "name": "go-ipfs-posinfo", "version": "0.0.1" + }, + "author": "hsanjuan", + "hash": "QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd", + "name": "go-ipfs-chunker", + "version": "0.0.1" } ], "gxVersion": "0.10.0", diff --git a/tar/format.go b/tar/format.go index 568300b34..8e7ac7323 100644 --- a/tar/format.go +++ b/tar/format.go @@ -9,11 +9,11 @@ import ( "strings" importer "github.com/ipfs/go-ipfs/importer" - chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" dagutil "github.com/ipfs/go-ipfs/merkledag/utils" path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index dfadd778b..dbeee7266 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -8,12 +8,12 @@ import ( "errors" "io" - chunk "github.com/ipfs/go-ipfs/importer/chunk" help "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" mdag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go index 0ca47c842..380c74f44 100644 --- a/unixfs/test/utils.go +++ b/unixfs/test/utils.go @@ -8,12 +8,12 @@ import ( "io/ioutil" "testing" - "github.com/ipfs/go-ipfs/importer/chunk" h "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" mdag "github.com/ipfs/go-ipfs/merkledag" mdagmock "github.com/ipfs/go-ipfs/merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" + "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"