Merge pull request #3919 from ipfs/kevina/add-hash-fun

adder: add support for specifying the hash function

core/commands/add.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 
 	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
 	blockservice "github.com/ipfs/go-ipfs/blockservice"
@@ -17,6 +18,7 @@ import (
 	mfs "github.com/ipfs/go-ipfs/mfs"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
+	mh "gx/ipfs/QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw/go-multihash"
 	u "gx/ipfs/QmWbjfz3u6HkAdPh34dgPchGbQjob6LXLhAeCGii2TX69n/go-ipfs-util"
 	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"
 )
@@ -39,6 +41,7 @@ const (
 	noCopyOptionName      = "nocopy"
 	fstoreCacheOptionName = "fscache"
 	cidVersionOptionName  = "cid-version"
+	hashOptionName        = "hash"
 )
 
 const adderOutChanSize = 8
@@ -90,6 +93,7 @@ You can now refer to the added file in a gateway, like so:
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmds.IntOption(cidVersionOptionName, "Cid version. Non-zero value will change default of 'raw-leaves' to true. (experimental)").Default(0),
+		cmds.StringOption(hashOptionName, "Hash function to use. Will set Cid version to 1 if used. (experimental)").Default("sha2-256"),
 	},
 	PreRun: func(req cmds.Request) error {
 		quiet, _, _ := req.Option(quietOptionName).Bool()
@@ -164,6 +168,7 @@ You can now refer to the added file in a gateway, like so:
 		nocopy, _, _ := req.Option(noCopyOptionName).Bool()
 		fscache, _, _ := req.Option(fstoreCacheOptionName).Bool()
 		cidVer, _, _ := req.Option(cidVersionOptionName).Int()
+		hashFunStr, hfset, _ := req.Option(hashOptionName).String()
 
 		if nocopy && !cfg.Experimental.FilestoreEnabled {
 			res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"),
@@ -180,6 +185,10 @@ You can now refer to the added file in a gateway, like so:
 			return
 		}
 
+		if hfset && cidVer == 0 {
+			cidVer = 1
+		}
+
 		if cidVer >= 1 && !rbset {
 			rawblks = true
 		}
@@ -190,6 +199,15 @@ You can now refer to the added file in a gateway, like so:
 			return
 		}
 
+		hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
+		if !ok {
+			res.SetError(fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)), cmds.ErrNormal)
+			return
+		}
+
+		prefix.MhType = hashFunCode
+		prefix.MhLength = -1
+
 		if hash {
 			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
 				//TODO: need this to be true or all files

importer/helpers/dagbuilder.go

@@ -165,10 +165,21 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
 	}
 
 	if db.rawLeaves {
-		return &UnixfsNode{
-			rawnode: dag.NewRawNode(data),
-			raw:     true,
-		}, nil
+		if db.prefix == nil {
+			return &UnixfsNode{
+				rawnode: dag.NewRawNode(data),
+				raw:     true,
+			}, nil
+		} else {
+			rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix)
+			if err != nil {
+				return nil, err
+			}
+			return &UnixfsNode{
+				rawnode: rawnode,
+				raw:     true,
+			}, nil
+		}
 	} else {
 		blk := db.NewUnixfsBlock()
 		blk.SetData(data)

merkledag/merkledag.go

@@ -116,7 +116,7 @@ func decodeBlock(b blocks.Block) (node.Node, error) {
 		decnd.Prefix = b.Cid().Prefix()
 		return decnd, nil
 	case cid.Raw:
-		return NewRawNode(b.RawData()), nil
+		return NewRawNodeWPrefix(b.RawData(), b.Cid().Prefix())
 	case cid.DagCBOR:
 		return ipldcbor.Decode(b.RawData())
 	default:

merkledag/raw.go

@@ -12,6 +12,8 @@ type RawNode struct {
 	blocks.Block
 }
 
+// NewRawNode creates a RawNode using the default sha2-256 hash
+// funcition.
 func NewRawNode(data []byte) *RawNode {
 	h := u.Hash(data)
 	c := cid.NewCidV1(cid.Raw, h)
@@ -20,6 +22,24 @@ func NewRawNode(data []byte) *RawNode {
 	return &RawNode{blk}
 }
 
+// NewRawNodeWPrefix creates a RawNode with the hash function
+// specified in prefix.
+func NewRawNodeWPrefix(data []byte, prefix cid.Prefix) (*RawNode, error) {
+	prefix.Codec = cid.Raw
+	if prefix.Version == 0 {
+		prefix.Version = 1
+	}
+	c, err := prefix.Sum(data)
+	if err != nil {
+		return nil, err
+	}
+	blk, err := blocks.NewBlockWithCid(data, c)
+	if err != nil {
+		return nil, err
+	}
+	return &RawNode{blk}, nil
+}
+
 func (rn *RawNode) Links() []*node.Link {
 	return nil
 }

test/sharness/t0040-add-and-cat.sh

@@ -403,6 +403,10 @@ MARS="zdj7WWx6fGNrNGkdpkuTAxCjKbQ1pPtarqA6VQhedhLTZu34J"
 VENUS="zdj7WbB1BUF8WejmVpQCmMLd1RbPnxJtvAj1Lep6eTmXRFbrz"
 add_directory '--cid-version=1 --raw-leaves=false'
 
+PLANETS="zDMZof1kqxDAx9myQbXsyWwyWP9qRPsXsWH7XuTz6isT7Rh1S6nM"
+MARS="zCT5htkdz1ZBHYVQXFQn51ngPXLVqaHSWoae87V1d6e9qWpSAjXw"
+VENUS="zCT5htke5JcdoMM4WhmUKXWf2QC3TnQToqGZHH1WsZERv6kPhFPg"
+add_directory '--hash=blake2b-256'
 
 test_expect_success "'ipfs add -rn' succeeds" '
 	mkdir -p mountdir/moons/jupiter &&
@@ -446,12 +450,26 @@ test_add_cat_5MB --cid-version=1 "zdj7WiiaedqVBXjX4SNqx3jfuZideDqdLYnDzCDJ66JDMK
 # format (i.e. not raw)
 test_add_cat_5MB '--cid-version=1 --raw-leaves=false' "zdj7WfgEsj897BBZj2mcfsRLhaPZcCixPV2G7DkWgF1Wdr64P"
 
+# note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true
+# the specified hash represents the leaf nodes stored as raw leaves and
+# encoded with the blake2b-256 hash funtion
+test_add_cat_5MB '--hash=blake2b-256' "zDMZof1kuxn7ebvKyvmkYLPvocSvFYxxAWT1yQBN1wWiXXr7w5mY"
+
+# the specified hash represents the leaf nodes stored as protoful nodes and
+# encoded with the blake2b-256 hash funtion
+test_add_cat_5MB '--hash=blake2b-256 --raw-leaves=false' "zDMZof1krz3SFTyhboRyWZyUP2qNgVdn9wjtaX211aHJ8WgeyT9v"
+
 test_add_cat_expensive "" "QmU9SWAPPmNEKZB8umYMmjYvN7VyHqABNvdA6GUi4MMEz3"
 
 # note: the specified hash implies that internal nodes are stored
 # using CidV1 and leaves are stored using raw blocks
 test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYhqxeJyW1wgCq"
 
+# note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true
+# the specified hash represents the leaf nodes stored as raw leaves and
+# encoded with the blake2b-256 hash funtion
+test_add_cat_expensive '--hash=blake2b-256' "zDMZof1kwndounDzQCANUHjiE3zt1mPEgx7RE3JTHoZrRRa79xcv"
+
 test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=true&r=true&stream-channels=true:"
 
 test_add_pwd_is_symlink