coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-02-23 19:37:46 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #5296 from ipfs/feat/extract-blockservice

Browse Source 
Extract blockservice and verifcid
This commit is contained in:
Whyrusleeping 2018-07-28 00:48:17 -07:00 committed by GitHub
commit 23f9e7bdb7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 28 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

336
blockservice/blockservice.go
View File

@ -1,336 +0,0 @@
// package blockservice implements a BlockService interface that provides
// a single GetBlock/AddBlock interface that seamlessly retrieves data either
// locally or from a remote peer through the exchange.
package blockservice
import (
"context"
"errors"
"fmt"
"io"
"github.com/ipfs/go-ipfs/thirdparty/verifcid"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
exchange "gx/ipfs/Qmc2faLf7URkHpsbfYM4EMbr8iSAcGAe8VPgVi64HVnwji/go-ipfs-exchange-interface"
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
)
var log = logging.Logger("blockservice")
var ErrNotFound = errors.New("blockservice: key not found")
// BlockGetter is the common interface shared between blockservice sessions and
// the blockservice.
type BlockGetter interface {
// GetBlock gets the requested block.
GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error)
// GetBlocks does a batch request for the given cids, returning blocks as
// they are found, in no particular order.
//
// It may not be able to find all requested blocks (or the context may
// be canceled). In that case, it will close the channel early. It is up
// to the consumer to detect this situation and keep track which blocks
// it has received and which it hasn't.
GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block
}
// BlockService is a hybrid block datastore. It stores data in a local
// datastore and may retrieve data from a remote Exchange.
// It uses an internal `datastore.Datastore` instance to store values.
type BlockService interface {
io.Closer
BlockGetter
// Blockstore returns a reference to the underlying blockstore
Blockstore() blockstore.Blockstore
// Exchange returns a reference to the underlying exchange (usually bitswap)
Exchange() exchange.Interface
// AddBlock puts a given block to the underlying datastore
AddBlock(o blocks.Block) error
// AddBlocks adds a slice of blocks at the same time using batching
// capabilities of the underlying datastore whenever possible.
AddBlocks(bs []blocks.Block) error
// DeleteBlock deletes the given block from the blockservice.
DeleteBlock(o *cid.Cid) error
}
type blockService struct {
blockstore blockstore.Blockstore
exchange exchange.Interface
// If checkFirst is true then first check that a block doesn't
// already exist to avoid republishing the block on the exchange.
checkFirst bool
}
// NewBlockService creates a BlockService with given datastore instance.
func New(bs blockstore.Blockstore, rem exchange.Interface) BlockService {
if rem == nil {
log.Warning("blockservice running in local (offline) mode.")
}
return &blockService{
blockstore: bs,
exchange: rem,
checkFirst: true,
}
}
// NewWriteThrough ceates a BlockService that guarantees writes will go
// through to the blockstore and are not skipped by cache checks.
func NewWriteThrough(bs blockstore.Blockstore, rem exchange.Interface) BlockService {
if rem == nil {
log.Warning("blockservice running in local (offline) mode.")
}
return &blockService{
blockstore: bs,
exchange: rem,
checkFirst: false,
}
}
// Blockstore returns the blockstore behind this blockservice.
func (s *blockService) Blockstore() blockstore.Blockstore {
return s.blockstore
}
// Exchange returns the exchange behind this blockservice.
func (s *blockService) Exchange() exchange.Interface {
return s.exchange
}
// NewSession creates a new session that allows for
// controlled exchange of wantlists to decrease the bandwidth overhead.
// If the current exchange is a SessionExchange, a new exchange
// session will be created. Otherwise, the current exchange will be used
// directly.
func NewSession(ctx context.Context, bs BlockService) *Session {
exch := bs.Exchange()
if sessEx, ok := exch.(exchange.SessionExchange); ok {
ses := sessEx.NewSession(ctx)
return &Session{
ses: ses,
bs: bs.Blockstore(),
}
}
return &Session{
ses: exch,
bs: bs.Blockstore(),
}
}
// AddBlock adds a particular block to the service, Putting it into the datastore.
// TODO pass a context into this if the remote.HasBlock is going to remain here.
func (s *blockService) AddBlock(o blocks.Block) error {
c := o.Cid()
// hash security
err := verifcid.ValidateCid(c)
if err != nil {
return err
}
if s.checkFirst {
if has, err := s.blockstore.Has(c); has || err != nil {
return err
}
}
if err := s.blockstore.Put(o); err != nil {
return err
}
log.Event(context.TODO(), "BlockService.BlockAdded", c)
if err := s.exchange.HasBlock(o); err != nil {
// TODO(#4623): really an error?
return errors.New("blockservice is closed")
}
return nil
}
func (s *blockService) AddBlocks(bs []blocks.Block) error {
// hash security
for _, b := range bs {
err := verifcid.ValidateCid(b.Cid())
if err != nil {
return err
}
}
var toput []blocks.Block
if s.checkFirst {
toput = make([]blocks.Block, 0, len(bs))
for _, b := range bs {
has, err := s.blockstore.Has(b.Cid())
if err != nil {
return err
}
if !has {
toput = append(toput, b)
}
}
} else {
toput = bs
}
err := s.blockstore.PutMany(toput)
if err != nil {
return err
}
for _, o := range toput {
log.Event(context.TODO(), "BlockService.BlockAdded", o.Cid())
if err := s.exchange.HasBlock(o); err != nil {
// TODO(#4623): Should this really *return*?
return fmt.Errorf("blockservice is closed (%s)", err)
}
}
return nil
}
// GetBlock retrieves a particular block from the service,
// Getting it from the datastore using the key (hash).
func (s *blockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) {
log.Debugf("BlockService GetBlock: '%s'", c)
var f exchange.Fetcher
if s.exchange != nil {
f = s.exchange
}
return getBlock(ctx, c, s.blockstore, f) // hash security
}
func getBlock(ctx context.Context, c *cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) (blocks.Block, error) {
err := verifcid.ValidateCid(c) // hash security
if err != nil {
return nil, err
}
block, err := bs.Get(c)
if err == nil {
return block, nil
}
if err == blockstore.ErrNotFound && f != nil {
// TODO be careful checking ErrNotFound. If the underlying
// implementation changes, this will break.
log.Debug("Blockservice: Searching bitswap")
blk, err := f.GetBlock(ctx, c)
if err != nil {
if err == blockstore.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}
log.Event(ctx, "BlockService.BlockFetched", c)
return blk, nil
}
log.Debug("Blockservice GetBlock: Not found")
if err == blockstore.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}
// GetBlocks gets a list of blocks asynchronously and returns through
// the returned channel.
// NB: No guarantees are made about order.
func (s *blockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block {
return getBlocks(ctx, ks, s.blockstore, s.exchange) // hash security
}
func getBlocks(ctx context.Context, ks []*cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) <-chan blocks.Block {
out := make(chan blocks.Block)
go func() {
defer close(out)
k := 0
for _, c := range ks {
// hash security
if err := verifcid.ValidateCid(c); err == nil {
ks[k] = c
k++
} else {
log.Errorf("unsafe CID (%s) passed to blockService.GetBlocks: %s", c, err)
}
}
ks = ks[:k]
var misses []*cid.Cid
for _, c := range ks {
hit, err := bs.Get(c)
if err != nil {
misses = append(misses, c)
continue
}
select {
case out <- hit:
case <-ctx.Done():
return
}
}
if len(misses) == 0 {
return
}
rblocks, err := f.GetBlocks(ctx, misses)
if err != nil {
log.Debugf("Error with GetBlocks: %s", err)
return
}
for b := range rblocks {
log.Event(ctx, "BlockService.BlockFetched", b.Cid())
select {
case out <- b:
case <-ctx.Done():
return
}
}
}()
return out
}
// DeleteBlock deletes a block in the blockservice from the datastore
func (s *blockService) DeleteBlock(c *cid.Cid) error {
err := s.blockstore.DeleteBlock(c)
if err == nil {
log.Event(context.TODO(), "BlockService.BlockDeleted", c)
}
return err
}
func (s *blockService) Close() error {
log.Debug("blockservice is shutting down...")
return s.exchange.Close()
}
// Session is a helper type to provide higher level access to bitswap sessions
type Session struct {
bs blockstore.Blockstore
ses exchange.Fetcher
}
// GetBlock gets a block in the context of a request session
func (s *Session) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) {
return getBlock(ctx, c, s.bs, s.ses) // hash security
}
// GetBlocks gets blocks in the context of a request session
func (s *Session) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block {
return getBlocks(ctx, ks, s.bs, s.ses) // hash security
}
var _ BlockGetter = (*Session)(nil)

48
blockservice/blockservice_test.go
View File

@ -1,48 +0,0 @@
package blockservice
import (
"testing"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
butil "gx/ipfs/QmYqPGpZ9Yemr55xus9DiEztkns6Jti5XJ7hC94JbvkdqZ/go-ipfs-blocksutil"
blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
dssync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
)
func TestWriteThroughWorks(t *testing.T) {
bstore := &PutCountingBlockstore{
blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())),
0,
}
bstore2 := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
exch := offline.Exchange(bstore2)
bserv := NewWriteThrough(bstore, exch)
bgen := butil.NewBlockGenerator()
block := bgen.Next()
t.Logf("PutCounter: %d", bstore.PutCounter)
bserv.AddBlock(block)
if bstore.PutCounter != 1 {
t.Fatalf("expected just one Put call, have: %d", bstore.PutCounter)
}
bserv.AddBlock(block)
if bstore.PutCounter != 2 {
t.Fatalf("Put should have called again, should be 2 is: %d", bstore.PutCounter)
}
}
var _ blockstore.Blockstore = (*PutCountingBlockstore)(nil)
type PutCountingBlockstore struct {
blockstore.Blockstore
PutCounter int
}
func (bs *PutCountingBlockstore) Put(block blocks.Block) error {
bs.PutCounter++
return bs.Blockstore.Put(block)
}

97
blockservice/test/blocks_test.go
View File

@ -1,97 +0,0 @@
package bstest
import (
"bytes"
"context"
"fmt"
"testing"
"time"
. "github.com/ipfs/go-ipfs/blockservice"
u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
dssync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
)
func newObject(data []byte) blocks.Block {
return blocks.NewBlock(data)
}
func TestBlocks(t *testing.T) {
bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
bs := New(bstore, offline.Exchange(bstore))
defer bs.Close()
o := newObject([]byte("beep boop"))
h := cid.NewCidV0(u.Hash([]byte("beep boop")))
if !o.Cid().Equals(h) {
t.Error("Block key and data multihash key not equal")
}
err := bs.AddBlock(o)
if err != nil {
t.Error("failed to add block to BlockService", err)
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
b2, err := bs.GetBlock(ctx, o.Cid())
if err != nil {
t.Error("failed to retrieve block from BlockService", err)
return
}
if !o.Cid().Equals(b2.Cid()) {
t.Error("Block keys not equal.")
}
if !bytes.Equal(o.RawData(), b2.RawData()) {
t.Error("Block data is not equal.")
}
}
func makeObjects(n int) []blocks.Block {
var out []blocks.Block
for i := 0; i < n; i++ {
out = append(out, newObject([]byte(fmt.Sprintf("object %d", i))))
}
return out
}
func TestGetBlocksSequential(t *testing.T) {
var servs = Mocks(4)
for _, s := range servs {
defer s.Close()
}
objs := makeObjects(50)
var cids []*cid.Cid
for _, o := range objs {
cids = append(cids, o.Cid())
servs[0].AddBlock(o)
}
t.Log("one instance at a time, get blocks concurrently")
for i := 1; i < len(servs); i++ {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*50)
defer cancel()
out := servs[i].GetBlocks(ctx, cids)
gotten := make(map[string]blocks.Block)
for blk := range out {
if _, ok := gotten[blk.Cid().KeyString()]; ok {
t.Fatal("Got duplicate block!")
}
gotten[blk.Cid().KeyString()] = blk
}
if len(gotten) != len(objs) {
t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs))
}
}
}

24
blockservice/test/mock.go
View File

@ -1,24 +0,0 @@
package bstest
import (
. "github.com/ipfs/go-ipfs/blockservice"
bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap"
tn "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/testnet"
delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
)
// Mocks returns |n| connected mock Blockservices
func Mocks(n int) []BlockService {
net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0))
sg := bitswap.NewTestSessionGenerator(net)
instances := sg.Instances(n)
var servs []BlockService
for _, i := range instances {
servs = append(servs, New(i.Blockstore(), i.Exchange))
}
return servs
}

2
core/builder.go
View File

@ -9,7 +9,6 @@ import (
"syscall"
"time"
bserv "github.com/ipfs/go-ipfs/blockservice"
filestore "github.com/ipfs/go-ipfs/filestore"
dag "github.com/ipfs/go-ipfs/merkledag"
resolver "github.com/ipfs/go-ipfs/path/resolver"
@ -18,6 +17,7 @@ import (
cfg "github.com/ipfs/go-ipfs/repo/config"
"github.com/ipfs/go-ipfs/thirdparty/verifbs"
uio "github.com/ipfs/go-ipfs/unixfs/io"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
goprocessctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context"

2
core/commands/add.go
View File

@ -7,7 +7,6 @@ import (
"os"
"strings"
blockservice "github.com/ipfs/go-ipfs/blockservice"
core "github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/core/coreunix"
filestore "github.com/ipfs/go-ipfs/filestore"
@ -15,6 +14,7 @@ import (
dagtest "github.com/ipfs/go-ipfs/merkledag/test"
mfs "github.com/ipfs/go-ipfs/mfs"
ft "github.com/ipfs/go-ipfs/unixfs"
blockservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"

2
core/commands/files.go
View File

@ -11,7 +11,6 @@ import (
"sort"
"strings"
bservice "github.com/ipfs/go-ipfs/blockservice"
oldcmds "github.com/ipfs/go-ipfs/commands"
lgc "github.com/ipfs/go-ipfs/commands/legacy"
core "github.com/ipfs/go-ipfs/core"
@ -22,6 +21,7 @@ import (
resolver "github.com/ipfs/go-ipfs/path/resolver"
ft "github.com/ipfs/go-ipfs/unixfs"
uio "github.com/ipfs/go-ipfs/unixfs/io"
bservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"

2
core/commands/ls.go
View File

@ -6,7 +6,6 @@ import (
"io"
"text/tabwriter"
blockservice "github.com/ipfs/go-ipfs/blockservice"
cmds "github.com/ipfs/go-ipfs/commands"
core "github.com/ipfs/go-ipfs/core"
e "github.com/ipfs/go-ipfs/core/commands/e"
@ -16,6 +15,7 @@ import (
unixfs "github.com/ipfs/go-ipfs/unixfs"
uio "github.com/ipfs/go-ipfs/unixfs/io"
unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
blockservice "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"

4
core/commands/pin.go
View File

@ -7,7 +7,6 @@ import (
"io"
"time"
bserv "github.com/ipfs/go-ipfs/blockservice"
cmds "github.com/ipfs/go-ipfs/commands"
core "github.com/ipfs/go-ipfs/core"
e "github.com/ipfs/go-ipfs/core/commands/e"
@ -16,10 +15,11 @@ import (
path "github.com/ipfs/go-ipfs/path"
resolver "github.com/ipfs/go-ipfs/path/resolver"
pin "github.com/ipfs/go-ipfs/pin"
"github.com/ipfs/go-ipfs/thirdparty/verifcid"
uio "github.com/ipfs/go-ipfs/unixfs/io"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
"gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
"gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"

2
core/core.go
View File

@ -20,7 +20,6 @@ import (
"strings"
"time"
bserv "github.com/ipfs/go-ipfs/blockservice"
rp "github.com/ipfs/go-ipfs/exchange/reprovide"
filestore "github.com/ipfs/go-ipfs/filestore"
mount "github.com/ipfs/go-ipfs/fuse/mount"
@ -34,6 +33,7 @@ import (
repo "github.com/ipfs/go-ipfs/repo"
config "github.com/ipfs/go-ipfs/repo/config"
ft "github.com/ipfs/go-ipfs/unixfs"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap"
bsnet "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/network"

2
core/coreapi/pin.go
View File

@ -4,11 +4,11 @@ import (
"context"
"fmt"
bserv "github.com/ipfs/go-ipfs/blockservice"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
corerepo "github.com/ipfs/go-ipfs/core/corerepo"
merkledag "github.com/ipfs/go-ipfs/merkledag"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"

2
core/coreunix/add_test.go
View File

@ -10,12 +10,12 @@ import (
"testing"
"time"
"github.com/ipfs/go-ipfs/blockservice"
"github.com/ipfs/go-ipfs/core"
dag "github.com/ipfs/go-ipfs/merkledag"
"github.com/ipfs/go-ipfs/pin/gc"
"github.com/ipfs/go-ipfs/repo"
"github.com/ipfs/go-ipfs/repo/config"
"gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
pi "gx/ipfs/QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L/go-ipfs-posinfo"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"

2
core/coreunix/metadata_test.go
View File

@ -6,12 +6,12 @@ import (
"io/ioutil"
"testing"
bserv "github.com/ipfs/go-ipfs/blockservice"
core "github.com/ipfs/go-ipfs/core"
importer "github.com/ipfs/go-ipfs/importer"
merkledag "github.com/ipfs/go-ipfs/merkledag"
ft "github.com/ipfs/go-ipfs/unixfs"
uio "github.com/ipfs/go-ipfs/unixfs/io"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"

3
exchange/reprovide/reprovide.go
View File

@ -5,9 +5,8 @@ import (
"fmt"
"time"
"github.com/ipfs/go-ipfs/thirdparty/verifcid"
backoff "gx/ipfs/QmPJUtEJsm5YLUWhF6imvyCH8KZXRJa9Wup7FDMwTy5Ufz/backoff"
"gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
routing "gx/ipfs/QmZ383TySJVeZWzGnWui6pRcKyYZk9VkKTuW7tmKRWk5au/go-libp2p-routing"
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"

2
merkledag/merkledag.go
View File

@ -6,7 +6,7 @@ import (
"fmt"
"sync"
bserv "github.com/ipfs/go-ipfs/blockservice"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
ipldcbor "gx/ipfs/QmWrbExtUaQQHjJ8FVVDAWj5o1MRAELDUV3VmoQsZHHb6L/go-ipld-cbor"

4
merkledag/merkledag_test.go
View File

@ -13,11 +13,11 @@ import (
"testing"
"time"
bserv "github.com/ipfs/go-ipfs/blockservice"
bstest "github.com/ipfs/go-ipfs/blockservice/test"
. "github.com/ipfs/go-ipfs/merkledag"
mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
dstest "github.com/ipfs/go-ipfs/merkledag/test"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
bstest "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice/test"
u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"

2
merkledag/test/utils.go
View File

@ -1,8 +1,8 @@
package mdutils
import (
bsrv "github.com/ipfs/go-ipfs/blockservice"
dag "github.com/ipfs/go-ipfs/merkledag"
bsrv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"

2
merkledag/utils/utils.go
View File

@ -4,9 +4,9 @@ import (
"context"
"errors"
bserv "github.com/ipfs/go-ipfs/blockservice"
dag "github.com/ipfs/go-ipfs/merkledag"
path "github.com/ipfs/go-ipfs/path"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"

2
mfs/mfs_test.go
View File

@ -14,12 +14,12 @@ import (
"testing"
"time"
bserv "github.com/ipfs/go-ipfs/blockservice"
importer "github.com/ipfs/go-ipfs/importer"
dag "github.com/ipfs/go-ipfs/merkledag"
"github.com/ipfs/go-ipfs/path"
ft "github.com/ipfs/go-ipfs/unixfs"
uio "github.com/ipfs/go-ipfs/unixfs/io"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"

6
package.json
View File

@ -551,6 +551,12 @@
"hash": "QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m",
"name": "go-bitswap",
"version": "1.0.0"
},
{
"author": "why",
"hash": "QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R",
"name": "go-blockservice",
"version": "1.0.1"
}
],
"gxVersion": "0.10.0",

4
pin/gc/gc.go
View File

@ -7,11 +7,11 @@ import (
"fmt"
"strings"
bserv "github.com/ipfs/go-ipfs/blockservice"
dag "github.com/ipfs/go-ipfs/merkledag"
pin "github.com/ipfs/go-ipfs/pin"
"github.com/ipfs/go-ipfs/thirdparty/verifcid"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
"gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"

2
pin/pin_test.go
View File

@ -5,8 +5,8 @@ import (
"testing"
"time"
bs "github.com/ipfs/go-ipfs/blockservice"
mdag "github.com/ipfs/go-ipfs/merkledag"
bs "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
util "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"

2
pin/set_test.go
View File

@ -5,8 +5,8 @@ import (
"encoding/binary"
"testing"
bserv "github.com/ipfs/go-ipfs/blockservice"
dag "github.com/ipfs/go-ipfs/merkledag"
bserv "gx/ipfs/QmNqRBAhovtf4jVd5cF7YvHaFSsQHHZBaUFwGQWPM2CV7R/go-blockservice"
offline "gx/ipfs/QmS6mo1dPpHdYsVkm27BRZDLxpKBCiJKUH8fHX15XFfMez/go-ipfs-exchange-offline"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"

3
thirdparty/verifbs/verifbs.go vendored
View File

@ -1,8 +1,7 @@
package verifbs
import (
"github.com/ipfs/go-ipfs/thirdparty/verifcid"
"gx/ipfs/QmQwgv79RHrRnoXmhnpC1BPtY55HHeneGMpPwmmBU1fUAG/go-verifcid"
blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
bstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"

62
thirdparty/verifcid/validate.go vendored
View File

@ -1,62 +0,0 @@
package verifcid
import (
"fmt"
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
)
var ErrPossiblyInsecureHashFunction = fmt.Errorf("potentially insecure hash functions not allowed")
var ErrBelowMinimumHashLength = fmt.Errorf("hashes must be at %d least bytes long", minimumHashLength)
const minimumHashLength = 20
var goodset = map[uint64]bool{
mh.SHA2_256: true,
mh.SHA2_512: true,
mh.SHA3_224: true,
mh.SHA3_256: true,
mh.SHA3_384: true,
mh.SHA3_512: true,
mh.SHAKE_256: true,
mh.DBL_SHA2_256: true,
mh.KECCAK_224: true,
mh.KECCAK_256: true,
mh.KECCAK_384: true,
mh.KECCAK_512: true,
mh.ID: true,
mh.SHA1: true, // not really secure but still useful
}
func IsGoodHash(code uint64) bool {
good, found := goodset[code]
if good {
return true
}
if !found {
if code >= mh.BLAKE2B_MIN+19 && code <= mh.BLAKE2B_MAX {
return true
}
if code >= mh.BLAKE2S_MIN+19 && code <= mh.BLAKE2S_MAX {
return true
}
}
return false
}
func ValidateCid(c *cid.Cid) error {
pref := c.Prefix()
if !IsGoodHash(pref.MhType) {
return ErrPossiblyInsecureHashFunction
}
if pref.MhType != mh.ID && pref.MhLength < minimumHashLength {
return ErrBelowMinimumHashLength
}
return nil
}

59
thirdparty/verifcid/validate_test.go vendored
View File

@ -1,59 +0,0 @@
package verifcid
import (
"testing"
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
)
func TestValidateCids(t *testing.T) {
assertTrue := func(v bool) {
t.Helper()
if !v {
t.Fatal("expected success")
}
}
assertFalse := func(v bool) {
t.Helper()
if v {
t.Fatal("expected failure")
}
}
assertTrue(IsGoodHash(mh.SHA2_256))
assertTrue(IsGoodHash(mh.BLAKE2B_MIN + 32))
assertTrue(IsGoodHash(mh.DBL_SHA2_256))
assertTrue(IsGoodHash(mh.KECCAK_256))
assertTrue(IsGoodHash(mh.SHA3))
assertTrue(IsGoodHash(mh.SHA1))
assertFalse(IsGoodHash(mh.BLAKE2B_MIN + 5))
mhcid := func(code uint64, length int) *cid.Cid {
mhash, err := mh.Sum([]byte{}, code, length)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.DagCBOR, mhash)
}
cases := []struct {
cid *cid.Cid
err error
}{
{mhcid(mh.SHA2_256, 32), nil},
{mhcid(mh.SHA2_256, 16), ErrBelowMinimumHashLength},
{mhcid(mh.MURMUR3, 4), ErrPossiblyInsecureHashFunction},
}
for i, cas := range cases {
if ValidateCid(cas.cid) != cas.err {
t.Errorf("wrong result in case of %s (index %d). Expected: %s, got %s",
cas.cid, i, cas.err, ValidateCid(cas.cid))
}
}
}