feat: switch to raw multihashes for blocks

Part of: https://github.com/ipfs/go-ipfs/issues/6815

core/commands/refs.go

@@ -136,7 +136,7 @@ var RefsLocalCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "List all local references.",
 		ShortDescription: `
-Displays the hashes of all local objects.
+Displays the hashes of all local objects. NOTE: This treats all local objects as "raw blocks" and returns CIDv1-Raw CIDs.
 `,
 	},
 

core/node/storage.go

@@ -9,7 +9,6 @@ import (
 	"github.com/ipfs/go-filestore"
 	"github.com/ipfs/go-ipfs/core/node/helpers"
 	"github.com/ipfs/go-ipfs/repo"
-	"github.com/ipfs/go-ipfs/thirdparty/cidv0v1"
 	"github.com/ipfs/go-ipfs/thirdparty/verifbs"
 )
 
@@ -41,7 +40,6 @@ func BaseBlockstoreCtor(cacheOpts blockstore.CacheOpts, nilRepo bool, hashOnRead
 		}
 
 		bs = blockstore.NewIdStore(bs)
-		bs = cidv0v1.NewBlockstore(bs)
 
 		if hashOnRead { // TODO: review: this is how it was done originally, is there a reason we can't just pass this directly?
 			bs.HashOnRead(true)

gc/gc.go

@@ -28,6 +28,16 @@ type Result struct {
 	Error      error
 }
 
+// converts a set of CIDs with different codecs to a set of CIDs with the raw codec.
+func toRawCids(set *cid.Set) *cid.Set {
+	newSet := cid.NewSet()
+	set.ForEach(func(c cid.Cid) error {
+		newSet.Add(cid.NewCidV1(cid.Raw, c.Hash()))
+		return nil
+	})
+	return newSet
+}
+
 // GC performs a mark and sweep garbage collection of the blocks in the blockstore
 // first, it creates a 'marked' set and adds to it the following:
 // - all recursively pinned blocks, plus all of their descendants (recursively)
@@ -60,6 +70,15 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
 			}
 			return
 		}
+
+		// The blockstore reports raw blocks. We need to remove the codecs from the CIDs.
+		gcs = toRawCids(gcs)
+		emark.Append(logging.LoggableMap{
+			"blackSetSize": fmt.Sprintf("%d", gcs.Len()),
+		})
+		emark.Done()
+		esweep := log.EventBegin(ctx, "GC.sweep")
+
 		keychan, err := bs.AllKeysChan(ctx)
 		if err != nil {
 			select {
@@ -79,6 +98,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
 				if !ok {
 					break loop
 				}
+				// NOTE: assumes that all CIDs returned by the keychan are _raw_ CIDv1 CIDs.
+				// This means we keep the block as long as we want it somewhere (CIDv1, CIDv0, Raw, other...).
 				if !gcs.Has(k) {
 					err := bs.DeleteBlock(ctx, k)
 					removed++
@@ -154,7 +175,9 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
 
 	for _, c := range roots {
 		// Walk recursively walks the dag and adds the keys to the given set
-		err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent())
+		err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool {
+			return set.Visit(toCidV1(k))
+		}, dag.Concurrent())
 
 		if err != nil {
 			err = verboseCidError(err)
@@ -165,6 +188,14 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
 	return nil
 }
 
+// toCidV1 converts any CIDv0s to CIDv1s.
+func toCidV1(c cid.Cid) cid.Cid {
+	if c.Version() == 0 {
+		return cid.NewCidV1(c.Type(), c.Hash())
+	}
+	return c
+}
+
 // ColoredSet computes the set of nodes in the graph that are pinned by the
 // pins in the given pinner.
 func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffortRoots []cid.Cid, output chan<- Result) (*cid.Set, error) {
@@ -225,7 +256,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
 		return nil, err
 	}
 	for _, k := range dkeys {
-		gcs.Add(k)
+		gcs.Add(toCidV1(k))
 	}
 
 	ikeys, err := pn.InternalPins(ctx)

thirdparty/cidv0v1/blockstore.go

@@ -1,89 +0,0 @@
-package cidv0v1
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	bs "github.com/ipfs/go-ipfs-blockstore"
-	mh "github.com/multiformats/go-multihash"
-)
-
-type blockstore struct {
-	bs.Blockstore
-}
-
-func NewBlockstore(b bs.Blockstore) bs.Blockstore {
-	return &blockstore{b}
-}
-
-func (b *blockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
-	have, err := b.Blockstore.Has(ctx, c)
-	if have || err != nil {
-		return have, err
-	}
-	c1 := tryOtherCidVersion(c)
-	if !c1.Defined() {
-		return false, nil
-	}
-	return b.Blockstore.Has(ctx, c1)
-}
-
-func (b *blockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
-	block, err := b.Blockstore.Get(ctx, c)
-	if err == nil {
-		return block, nil
-	}
-	if err != bs.ErrNotFound {
-		return nil, err
-	}
-	c1 := tryOtherCidVersion(c)
-	if !c1.Defined() {
-		return nil, bs.ErrNotFound
-	}
-	block, err = b.Blockstore.Get(ctx, c1)
-	if err != nil {
-		return nil, err
-	}
-	// modify block so it has the original CID
-	block, err = blocks.NewBlockWithCid(block.RawData(), c)
-	if err != nil {
-		return nil, err
-	}
-	// insert the block with the original CID to avoid problems
-	// with pinning
-	err = b.Blockstore.Put(ctx, block)
-	if err != nil {
-		return nil, err
-	}
-	return block, nil
-}
-
-func (b *blockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
-	size, err := b.Blockstore.GetSize(ctx, c)
-	if err == nil {
-		return size, nil
-	}
-	if err != bs.ErrNotFound {
-		return -1, err
-	}
-	c1 := tryOtherCidVersion(c)
-	if !c1.Defined() {
-		return -1, bs.ErrNotFound
-	}
-	return b.Blockstore.GetSize(ctx, c1)
-}
-
-func tryOtherCidVersion(c cid.Cid) cid.Cid {
-	prefix := c.Prefix()
-	if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 {
-		return cid.Undef
-	}
-	var c1 cid.Cid
-	if prefix.Version == 0 {
-		c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash())
-	} else {
-		c1 = cid.NewCidV0(c.Hash())
-	}
-	return c1
-}