From 0a3a3c9ba717b3b1d33147fc80d9318c4e0c1bcb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 27 Jul 2018 18:21:06 -0700 Subject: [PATCH] Extract blockservice and verifcid License: MIT Signed-off-by: Jeromy --- blockservice/blockservice.go | 336 --------------------------- blockservice/blockservice_test.go | 48 ---- blockservice/test/blocks_test.go | 97 -------- blockservice/test/mock.go | 24 -- core/builder.go | 2 +- core/commands/add.go | 2 +- core/commands/files.go | 2 +- core/commands/ls.go | 2 +- core/commands/pin.go | 4 +- core/core.go | 2 +- core/coreapi/pin.go | 2 +- core/coreunix/add_test.go | 2 +- core/coreunix/metadata_test.go | 2 +- exchange/reprovide/reprovide.go | 3 +- merkledag/merkledag.go | 2 +- merkledag/merkledag_test.go | 4 +- merkledag/test/utils.go | 2 +- merkledag/utils/utils.go | 2 +- mfs/mfs_test.go | 2 +- package.json | 6 + pin/gc/gc.go | 4 +- pin/pin_test.go | 2 +- pin/set_test.go | 2 +- thirdparty/verifbs/verifbs.go | 3 +- thirdparty/verifcid/validate.go | 62 ----- thirdparty/verifcid/validate_test.go | 59 ----- 26 files changed, 28 insertions(+), 650 deletions(-) delete mode 100644 blockservice/blockservice.go delete mode 100644 blockservice/blockservice_test.go delete mode 100644 blockservice/test/blocks_test.go delete mode 100644 blockservice/test/mock.go delete mode 100644 thirdparty/verifcid/validate.go delete mode 100644 thirdparty/verifcid/validate_test.go diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go deleted file mode 100644 index c4502c975..000000000 --- a/blockservice/blockservice.go +++ /dev/null @@ -1,336 +0,0 @@ -// package blockservice implements a BlockService interface that provides -// a single GetBlock/AddBlock interface that seamlessly retrieves data either -// locally or from a remote peer through the exchange. -package blockservice - -import ( - "context" - "errors" - "fmt" - "io" - - "github.com/ipfs/go-ipfs/thirdparty/verifcid" - - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" - exchange "gx/ipfs/Qmc2faLf7URkHpsbfYM4EMbr8iSAcGAe8VPgVi64HVnwji/go-ipfs-exchange-interface" - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" -) - -var log = logging.Logger("blockservice") - -var ErrNotFound = errors.New("blockservice: key not found") - -// BlockGetter is the common interface shared between blockservice sessions and -// the blockservice. -type BlockGetter interface { - // GetBlock gets the requested block. - GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) - - // GetBlocks does a batch request for the given cids, returning blocks as - // they are found, in no particular order. - // - // It may not be able to find all requested blocks (or the context may - // be canceled). In that case, it will close the channel early. It is up - // to the consumer to detect this situation and keep track which blocks - // it has received and which it hasn't. - GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block -} - -// BlockService is a hybrid block datastore. It stores data in a local -// datastore and may retrieve data from a remote Exchange. -// It uses an internal `datastore.Datastore` instance to store values. -type BlockService interface { - io.Closer - BlockGetter - - // Blockstore returns a reference to the underlying blockstore - Blockstore() blockstore.Blockstore - - // Exchange returns a reference to the underlying exchange (usually bitswap) - Exchange() exchange.Interface - - // AddBlock puts a given block to the underlying datastore - AddBlock(o blocks.Block) error - - // AddBlocks adds a slice of blocks at the same time using batching - // capabilities of the underlying datastore whenever possible. - AddBlocks(bs []blocks.Block) error - - // DeleteBlock deletes the given block from the blockservice. - DeleteBlock(o *cid.Cid) error -} - -type blockService struct { - blockstore blockstore.Blockstore - exchange exchange.Interface - // If checkFirst is true then first check that a block doesn't - // already exist to avoid republishing the block on the exchange. - checkFirst bool -} - -// NewBlockService creates a BlockService with given datastore instance. -func New(bs blockstore.Blockstore, rem exchange.Interface) BlockService { - if rem == nil { - log.Warning("blockservice running in local (offline) mode.") - } - - return &blockService{ - blockstore: bs, - exchange: rem, - checkFirst: true, - } -} - -// NewWriteThrough ceates a BlockService that guarantees writes will go -// through to the blockstore and are not skipped by cache checks. -func NewWriteThrough(bs blockstore.Blockstore, rem exchange.Interface) BlockService { - if rem == nil { - log.Warning("blockservice running in local (offline) mode.") - } - - return &blockService{ - blockstore: bs, - exchange: rem, - checkFirst: false, - } -} - -// Blockstore returns the blockstore behind this blockservice. -func (s *blockService) Blockstore() blockstore.Blockstore { - return s.blockstore -} - -// Exchange returns the exchange behind this blockservice. -func (s *blockService) Exchange() exchange.Interface { - return s.exchange -} - -// NewSession creates a new session that allows for -// controlled exchange of wantlists to decrease the bandwidth overhead. -// If the current exchange is a SessionExchange, a new exchange -// session will be created. Otherwise, the current exchange will be used -// directly. -func NewSession(ctx context.Context, bs BlockService) *Session { - exch := bs.Exchange() - if sessEx, ok := exch.(exchange.SessionExchange); ok { - ses := sessEx.NewSession(ctx) - return &Session{ - ses: ses, - bs: bs.Blockstore(), - } - } - return &Session{ - ses: exch, - bs: bs.Blockstore(), - } -} - -// AddBlock adds a particular block to the service, Putting it into the datastore. -// TODO pass a context into this if the remote.HasBlock is going to remain here. -func (s *blockService) AddBlock(o blocks.Block) error { - c := o.Cid() - // hash security - err := verifcid.ValidateCid(c) - if err != nil { - return err - } - if s.checkFirst { - if has, err := s.blockstore.Has(c); has || err != nil { - return err - } - } - - if err := s.blockstore.Put(o); err != nil { - return err - } - - log.Event(context.TODO(), "BlockService.BlockAdded", c) - - if err := s.exchange.HasBlock(o); err != nil { - // TODO(#4623): really an error? - return errors.New("blockservice is closed") - } - - return nil -} - -func (s *blockService) AddBlocks(bs []blocks.Block) error { - // hash security - for _, b := range bs { - err := verifcid.ValidateCid(b.Cid()) - if err != nil { - return err - } - } - var toput []blocks.Block - if s.checkFirst { - toput = make([]blocks.Block, 0, len(bs)) - for _, b := range bs { - has, err := s.blockstore.Has(b.Cid()) - if err != nil { - return err - } - if !has { - toput = append(toput, b) - } - } - } else { - toput = bs - } - - err := s.blockstore.PutMany(toput) - if err != nil { - return err - } - - for _, o := range toput { - log.Event(context.TODO(), "BlockService.BlockAdded", o.Cid()) - if err := s.exchange.HasBlock(o); err != nil { - // TODO(#4623): Should this really *return*? - return fmt.Errorf("blockservice is closed (%s)", err) - } - } - return nil -} - -// GetBlock retrieves a particular block from the service, -// Getting it from the datastore using the key (hash). -func (s *blockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) { - log.Debugf("BlockService GetBlock: '%s'", c) - - var f exchange.Fetcher - if s.exchange != nil { - f = s.exchange - } - - return getBlock(ctx, c, s.blockstore, f) // hash security -} - -func getBlock(ctx context.Context, c *cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) (blocks.Block, error) { - err := verifcid.ValidateCid(c) // hash security - if err != nil { - return nil, err - } - - block, err := bs.Get(c) - if err == nil { - return block, nil - } - - if err == blockstore.ErrNotFound && f != nil { - // TODO be careful checking ErrNotFound. If the underlying - // implementation changes, this will break. - log.Debug("Blockservice: Searching bitswap") - blk, err := f.GetBlock(ctx, c) - if err != nil { - if err == blockstore.ErrNotFound { - return nil, ErrNotFound - } - return nil, err - } - log.Event(ctx, "BlockService.BlockFetched", c) - return blk, nil - } - - log.Debug("Blockservice GetBlock: Not found") - if err == blockstore.ErrNotFound { - return nil, ErrNotFound - } - - return nil, err -} - -// GetBlocks gets a list of blocks asynchronously and returns through -// the returned channel. -// NB: No guarantees are made about order. -func (s *blockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block { - return getBlocks(ctx, ks, s.blockstore, s.exchange) // hash security -} - -func getBlocks(ctx context.Context, ks []*cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) <-chan blocks.Block { - out := make(chan blocks.Block) - - go func() { - defer close(out) - - k := 0 - for _, c := range ks { - // hash security - if err := verifcid.ValidateCid(c); err == nil { - ks[k] = c - k++ - } else { - log.Errorf("unsafe CID (%s) passed to blockService.GetBlocks: %s", c, err) - } - } - ks = ks[:k] - - var misses []*cid.Cid - for _, c := range ks { - hit, err := bs.Get(c) - if err != nil { - misses = append(misses, c) - continue - } - select { - case out <- hit: - case <-ctx.Done(): - return - } - } - - if len(misses) == 0 { - return - } - - rblocks, err := f.GetBlocks(ctx, misses) - if err != nil { - log.Debugf("Error with GetBlocks: %s", err) - return - } - - for b := range rblocks { - log.Event(ctx, "BlockService.BlockFetched", b.Cid()) - select { - case out <- b: - case <-ctx.Done(): - return - } - } - }() - return out -} - -// DeleteBlock deletes a block in the blockservice from the datastore -func (s *blockService) DeleteBlock(c *cid.Cid) error { - err := s.blockstore.DeleteBlock(c) - if err == nil { - log.Event(context.TODO(), "BlockService.BlockDeleted", c) - } - return err -} - -func (s *blockService) Close() error { - log.Debug("blockservice is shutting down...") - return s.exchange.Close() -} - -// Session is a helper type to provide higher level access to bitswap sessions -type Session struct { - bs blockstore.Blockstore - ses exchange.Fetcher -} - -// GetBlock gets a block in the context of a request session -func (s *Session) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) { - return getBlock(ctx, c, s.bs, s.ses) // hash security -} - -// GetBlocks gets blocks in the context of a request session -func (s *Session) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block { - return getBlocks(ctx, ks, s.bs, s.ses) // hash security -} - -var _ BlockGetter = (*Session)(nil) diff --git a/blockservice/blockservice_test.go b/blockservice/blockservice_test.go deleted file mode 100644 index e94f4a3fe..000000000 --- a/blockservice/blockservice_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package blockservice - -import ( - "testing" - - offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" - butil "gx/ipfs/QmYqPGpZ9Yemr55xus9DiEztkns6Jti5XJ7hC94JbvkdqZ/go-ipfs-blocksutil" - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" - ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" - dssync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync" -) - -func TestWriteThroughWorks(t *testing.T) { - bstore := &PutCountingBlockstore{ - blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())), - 0, - } - bstore2 := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - exch := offline.Exchange(bstore2) - bserv := NewWriteThrough(bstore, exch) - bgen := butil.NewBlockGenerator() - - block := bgen.Next() - - t.Logf("PutCounter: %d", bstore.PutCounter) - bserv.AddBlock(block) - if bstore.PutCounter != 1 { - t.Fatalf("expected just one Put call, have: %d", bstore.PutCounter) - } - - bserv.AddBlock(block) - if bstore.PutCounter != 2 { - t.Fatalf("Put should have called again, should be 2 is: %d", bstore.PutCounter) - } -} - -var _ blockstore.Blockstore = (*PutCountingBlockstore)(nil) - -type PutCountingBlockstore struct { - blockstore.Blockstore - PutCounter int -} - -func (bs *PutCountingBlockstore) Put(block blocks.Block) error { - bs.PutCounter++ - return bs.Blockstore.Put(block) -} diff --git a/blockservice/test/blocks_test.go b/blockservice/test/blocks_test.go deleted file mode 100644 index 19a2e54f0..000000000 --- a/blockservice/test/blocks_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package bstest - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - . "github.com/ipfs/go-ipfs/blockservice" - - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" - offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" - ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" - dssync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync" -) - -func newObject(data []byte) blocks.Block { - return blocks.NewBlock(data) -} - -func TestBlocks(t *testing.T) { - bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - bs := New(bstore, offline.Exchange(bstore)) - defer bs.Close() - - o := newObject([]byte("beep boop")) - h := cid.NewCidV0(u.Hash([]byte("beep boop"))) - if !o.Cid().Equals(h) { - t.Error("Block key and data multihash key not equal") - } - - err := bs.AddBlock(o) - if err != nil { - t.Error("failed to add block to BlockService", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - b2, err := bs.GetBlock(ctx, o.Cid()) - if err != nil { - t.Error("failed to retrieve block from BlockService", err) - return - } - - if !o.Cid().Equals(b2.Cid()) { - t.Error("Block keys not equal.") - } - - if !bytes.Equal(o.RawData(), b2.RawData()) { - t.Error("Block data is not equal.") - } -} - -func makeObjects(n int) []blocks.Block { - var out []blocks.Block - for i := 0; i < n; i++ { - out = append(out, newObject([]byte(fmt.Sprintf("object %d", i)))) - } - return out -} - -func TestGetBlocksSequential(t *testing.T) { - var servs = Mocks(4) - for _, s := range servs { - defer s.Close() - } - objs := makeObjects(50) - - var cids []*cid.Cid - for _, o := range objs { - cids = append(cids, o.Cid()) - servs[0].AddBlock(o) - } - - t.Log("one instance at a time, get blocks concurrently") - - for i := 1; i < len(servs); i++ { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) - defer cancel() - out := servs[i].GetBlocks(ctx, cids) - gotten := make(map[string]blocks.Block) - for blk := range out { - if _, ok := gotten[blk.Cid().KeyString()]; ok { - t.Fatal("Got duplicate block!") - } - gotten[blk.Cid().KeyString()] = blk - } - if len(gotten) != len(objs) { - t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs)) - } - } -} diff --git a/blockservice/test/mock.go b/blockservice/test/mock.go deleted file mode 100644 index 831bb8cfb..000000000 --- a/blockservice/test/mock.go +++ /dev/null @@ -1,24 +0,0 @@ -package bstest - -import ( - . "github.com/ipfs/go-ipfs/blockservice" - bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap" - tn "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/testnet" - - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay" - mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock" -) - -// Mocks returns |n| connected mock Blockservices -func Mocks(n int) []BlockService { - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0)) - sg := bitswap.NewTestSessionGenerator(net) - - instances := sg.Instances(n) - - var servs []BlockService - for _, i := range instances { - servs = append(servs, New(i.Blockstore(), i.Exchange)) - } - return servs -} diff --git a/core/builder.go b/core/builder.go index d1fa89612..26dfa2d22 100644 --- a/core/builder.go +++ b/core/builder.go @@ -9,7 +9,6 @@ import ( "syscall" "time" - bserv "github.com/ipfs/go-ipfs/blockservice" filestore "github.com/ipfs/go-ipfs/filestore" dag "github.com/ipfs/go-ipfs/merkledag" resolver "github.com/ipfs/go-ipfs/path/resolver" @@ -18,6 +17,7 @@ import ( cfg "github.com/ipfs/go-ipfs/repo/config" "github.com/ipfs/go-ipfs/thirdparty/verifbs" uio "github.com/ipfs/go-ipfs/unixfs/io" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" goprocessctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context" diff --git a/core/commands/add.go b/core/commands/add.go index 3925c81d4..492d4dc7f 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -7,7 +7,6 @@ import ( "os" "strings" - blockservice "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreunix" filestore "github.com/ipfs/go-ipfs/filestore" @@ -15,6 +14,7 @@ import ( dagtest "github.com/ipfs/go-ipfs/merkledag/test" mfs "github.com/ipfs/go-ipfs/mfs" ft "github.com/ipfs/go-ipfs/unixfs" + blockservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" diff --git a/core/commands/files.go b/core/commands/files.go index edfdd8b19..143c5484f 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -11,7 +11,6 @@ import ( "sort" "strings" - bservice "github.com/ipfs/go-ipfs/blockservice" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" @@ -22,6 +21,7 @@ import ( resolver "github.com/ipfs/go-ipfs/path/resolver" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + bservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" diff --git a/core/commands/ls.go b/core/commands/ls.go index 332d3b851..0ed5c2633 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,7 +6,6 @@ import ( "io" "text/tabwriter" - blockservice "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" @@ -16,6 +15,7 @@ import ( unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" + blockservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" diff --git a/core/commands/pin.go b/core/commands/pin.go index 199bbab9a..0ec7c0d72 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -7,7 +7,6 @@ import ( "io" "time" - bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" @@ -16,10 +15,11 @@ import ( path "github.com/ipfs/go-ipfs/path" resolver "github.com/ipfs/go-ipfs/path/resolver" pin "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" uio "github.com/ipfs/go-ipfs/unixfs/io" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" + "gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" diff --git a/core/core.go b/core/core.go index df18071e5..5942e73b3 100644 --- a/core/core.go +++ b/core/core.go @@ -20,7 +20,6 @@ import ( "strings" "time" - bserv "github.com/ipfs/go-ipfs/blockservice" rp "github.com/ipfs/go-ipfs/exchange/reprovide" filestore "github.com/ipfs/go-ipfs/filestore" mount "github.com/ipfs/go-ipfs/fuse/mount" @@ -34,6 +33,7 @@ import ( repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" ft "github.com/ipfs/go-ipfs/unixfs" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap" bsnet "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/network" diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 1f767ee4f..2dc767cd8 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - bserv "github.com/ipfs/go-ipfs/blockservice" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" corerepo "github.com/ipfs/go-ipfs/core/corerepo" merkledag "github.com/ipfs/go-ipfs/merkledag" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index a783e2236..96bfcbe6a 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" - "github.com/ipfs/go-ipfs/blockservice" "github.com/ipfs/go-ipfs/core" dag "github.com/ipfs/go-ipfs/merkledag" "github.com/ipfs/go-ipfs/pin/gc" "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo/config" + "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" pi "gx/ipfs/QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L/go-ipfs-posinfo" blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go index 92e009945..8d2c1fecd 100644 --- a/core/coreunix/metadata_test.go +++ b/core/coreunix/metadata_test.go @@ -6,12 +6,12 @@ import ( "io/ioutil" "testing" - bserv "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" importer "github.com/ipfs/go-ipfs/importer" merkledag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" diff --git a/exchange/reprovide/reprovide.go b/exchange/reprovide/reprovide.go index c77f7679d..c92f91595 100644 --- a/exchange/reprovide/reprovide.go +++ b/exchange/reprovide/reprovide.go @@ -5,9 +5,8 @@ import ( "fmt" "time" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" - backoff "gx/ipfs/QmPJUtEJsm5YLUWhF6imvyCH8KZXRJa9Wup7FDMwTy5Ufz/backoff" + "gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" routing "gx/ipfs/QmZ383TySJVeZWzGnWui6pRcKyYZk9VkKTuW7tmKRWk5au/go-libp2p-routing" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 4bdf734e8..5ac49dbd4 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -6,7 +6,7 @@ import ( "fmt" "sync" - bserv "github.com/ipfs/go-ipfs/blockservice" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" ipldcbor "gx/ipfs/QmWrbExtUaQQHjJ8FVVDAWj5o1MRAELDUV3VmoQsZHHb6L/go-ipld-cbor" diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 1153b6989..342ad91fa 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -13,11 +13,11 @@ import ( "testing" "time" - bserv "github.com/ipfs/go-ipfs/blockservice" - bstest "github.com/ipfs/go-ipfs/blockservice/test" . "github.com/ipfs/go-ipfs/merkledag" mdpb "github.com/ipfs/go-ipfs/merkledag/pb" dstest "github.com/ipfs/go-ipfs/merkledag/test" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" + bstest "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice/test" u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" diff --git a/merkledag/test/utils.go b/merkledag/test/utils.go index c161f7c01..7e26a5cf5 100644 --- a/merkledag/test/utils.go +++ b/merkledag/test/utils.go @@ -1,8 +1,8 @@ package mdutils import ( - bsrv "github.com/ipfs/go-ipfs/blockservice" dag "github.com/ipfs/go-ipfs/merkledag" + bsrv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format" diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go index fbd18c725..1937ba10b 100644 --- a/merkledag/utils/utils.go +++ b/merkledag/utils/utils.go @@ -4,9 +4,9 @@ import ( "context" "errors" - bserv "github.com/ipfs/go-ipfs/blockservice" dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format" diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index d2f22bcd4..e1af70d15 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -14,12 +14,12 @@ import ( "testing" "time" - bserv "github.com/ipfs/go-ipfs/blockservice" importer "github.com/ipfs/go-ipfs/importer" dag "github.com/ipfs/go-ipfs/merkledag" "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" diff --git a/package.json b/package.json index 66903c26f..82cabc07a 100644 --- a/package.json +++ b/package.json @@ -551,6 +551,12 @@ "hash": "QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m", "name": "go-bitswap", "version": "1.0.0" + }, + { + "author": "why", + "hash": "QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R", + "name": "go-blockservice", + "version": "1.0.1" } ], "gxVersion": "0.10.0", diff --git a/pin/gc/gc.go b/pin/gc/gc.go index e01321c65..eb30016ee 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -7,11 +7,11 @@ import ( "fmt" "strings" - bserv "github.com/ipfs/go-ipfs/blockservice" dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" + "gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format" diff --git a/pin/pin_test.go b/pin/pin_test.go index f832e0a72..ae6ace111 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - bs "github.com/ipfs/go-ipfs/blockservice" mdag "github.com/ipfs/go-ipfs/merkledag" + bs "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" util "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" diff --git a/pin/set_test.go b/pin/set_test.go index b2fe149a4..c477a71d7 100644 --- a/pin/set_test.go +++ b/pin/set_test.go @@ -5,8 +5,8 @@ import ( "encoding/binary" "testing" - bserv "github.com/ipfs/go-ipfs/blockservice" dag "github.com/ipfs/go-ipfs/merkledag" + bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice" offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" diff --git a/thirdparty/verifbs/verifbs.go b/thirdparty/verifbs/verifbs.go index 9052f756f..73edb7edf 100644 --- a/thirdparty/verifbs/verifbs.go +++ b/thirdparty/verifbs/verifbs.go @@ -1,8 +1,7 @@ package verifbs import ( - "github.com/ipfs/go-ipfs/thirdparty/verifcid" - + "gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid" blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" bstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" diff --git a/thirdparty/verifcid/validate.go b/thirdparty/verifcid/validate.go deleted file mode 100644 index 6e4c80d61..000000000 --- a/thirdparty/verifcid/validate.go +++ /dev/null @@ -1,62 +0,0 @@ -package verifcid - -import ( - "fmt" - - mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" -) - -var ErrPossiblyInsecureHashFunction = fmt.Errorf("potentially insecure hash functions not allowed") -var ErrBelowMinimumHashLength = fmt.Errorf("hashes must be at %d least bytes long", minimumHashLength) - -const minimumHashLength = 20 - -var goodset = map[uint64]bool{ - mh.SHA2_256: true, - mh.SHA2_512: true, - mh.SHA3_224: true, - mh.SHA3_256: true, - mh.SHA3_384: true, - mh.SHA3_512: true, - mh.SHAKE_256: true, - mh.DBL_SHA2_256: true, - mh.KECCAK_224: true, - mh.KECCAK_256: true, - mh.KECCAK_384: true, - mh.KECCAK_512: true, - mh.ID: true, - - mh.SHA1: true, // not really secure but still useful -} - -func IsGoodHash(code uint64) bool { - good, found := goodset[code] - if good { - return true - } - - if !found { - if code >= mh.BLAKE2B_MIN+19 && code <= mh.BLAKE2B_MAX { - return true - } - if code >= mh.BLAKE2S_MIN+19 && code <= mh.BLAKE2S_MAX { - return true - } - } - - return false -} - -func ValidateCid(c *cid.Cid) error { - pref := c.Prefix() - if !IsGoodHash(pref.MhType) { - return ErrPossiblyInsecureHashFunction - } - - if pref.MhType != mh.ID && pref.MhLength < minimumHashLength { - return ErrBelowMinimumHashLength - } - - return nil -} diff --git a/thirdparty/verifcid/validate_test.go b/thirdparty/verifcid/validate_test.go deleted file mode 100644 index a058f312c..000000000 --- a/thirdparty/verifcid/validate_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package verifcid - -import ( - "testing" - - mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" - - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" -) - -func TestValidateCids(t *testing.T) { - assertTrue := func(v bool) { - t.Helper() - if !v { - t.Fatal("expected success") - } - } - assertFalse := func(v bool) { - t.Helper() - if v { - t.Fatal("expected failure") - } - } - - assertTrue(IsGoodHash(mh.SHA2_256)) - assertTrue(IsGoodHash(mh.BLAKE2B_MIN + 32)) - assertTrue(IsGoodHash(mh.DBL_SHA2_256)) - assertTrue(IsGoodHash(mh.KECCAK_256)) - assertTrue(IsGoodHash(mh.SHA3)) - - assertTrue(IsGoodHash(mh.SHA1)) - - assertFalse(IsGoodHash(mh.BLAKE2B_MIN + 5)) - - mhcid := func(code uint64, length int) *cid.Cid { - mhash, err := mh.Sum([]byte{}, code, length) - if err != nil { - t.Fatal(err) - } - return cid.NewCidV1(cid.DagCBOR, mhash) - } - - cases := []struct { - cid *cid.Cid - err error - }{ - {mhcid(mh.SHA2_256, 32), nil}, - {mhcid(mh.SHA2_256, 16), ErrBelowMinimumHashLength}, - {mhcid(mh.MURMUR3, 4), ErrPossiblyInsecureHashFunction}, - } - - for i, cas := range cases { - if ValidateCid(cas.cid) != cas.err { - t.Errorf("wrong result in case of %s (index %d). Expected: %s, got %s", - cas.cid, i, cas.err, ValidateCid(cas.cid)) - } - } - -}