diff --git a/core/commands/add.go b/core/commands/add.go index ba0dc2ecb..d08fd0834 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -45,6 +45,7 @@ const ( hashOptionName = "hash" inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" + rawFileHashName = "raw-file-hash" ) const adderOutChanSize = 8 @@ -140,6 +141,7 @@ only-hash, and progress/status related flags) will change the final hash. cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"), cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), + cmds.StringOption(rawFileHashName, "Additionally reference and share files using the hash of the full file. Takes the name of a hash function (experimental)"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -180,6 +182,7 @@ only-hash, and progress/status related flags) will change the final hash. hashFunStr, _ := req.Options[hashOptionName].(string) inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) + rawFileHashStr, rawFileSet := req.Options[rawFileHashName].(string) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { @@ -227,6 +230,15 @@ only-hash, and progress/status related flags) will change the final hash. opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } + if rawFileSet { + var rawHashFunCode uint64 + rawHashFunCode, ok = mh.Names[strings.ToLower(rawFileHashStr)] + if !ok { + return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(rawFileHashStr)) + } + opts = append(opts, options.Unixfs.RawFileHash(rawHashFunCode)) + } + opts = append(opts, nil) // events option placeholder var added int diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 53d13a3a4..631fec37f 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -151,6 +151,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options } } + if settings.RawFileHashSet { + fileAdder.RawFileHash = &settings.RawFileHash + } + if settings.OnlyHash { md := dagtest.Mock() emptyDirNode := ft.EmptyDirNode() diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 7ac4b06fa..19df92a61 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -2,8 +2,10 @@ package coreunix import ( "context" + "crypto/sha256" "errors" "fmt" + "github.com/multiformats/go-multihash" "io" gopath "path" "strconv" @@ -61,24 +63,25 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG // Adder holds the switches passed to the `add` command. type Adder struct { - ctx context.Context - pinning pin.Pinner - gcLocker bstore.GCLocker - dagService ipld.DAGService - bufferedDS *ipld.BufferedDAG - Out chan<- interface{} - Progress bool - Pin bool - Trickle bool - RawLeaves bool - Silent bool - NoCopy bool - Chunker string - mroot *mfs.Root - unlocker bstore.Unlocker - tempRoot cid.Cid - CidBuilder cid.Builder - liveNodes uint64 + ctx context.Context + pinning pin.Pinner + gcLocker bstore.GCLocker + dagService ipld.DAGService + bufferedDS *ipld.BufferedDAG + Out chan<- interface{} + Progress bool + Pin bool + Trickle bool + RawLeaves bool + Silent bool + NoCopy bool + RawFileHash *uint64 + Chunker string + mroot *mfs.Root + unlocker bstore.Unlocker + tempRoot cid.Cid + CidBuilder cid.Builder + liveNodes uint64 } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -101,10 +104,10 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) { } // Constructs a node from reader's data, and adds it. Doesn't pin. -func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error) { +func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { chnk, err := chunker.FromString(reader, adder.Chunker) if err != nil { - return nil, nil, err + return nil, err } params := ihelper.DagBuilderParams{ @@ -117,7 +120,7 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error) db, err := params.New(chnk) if err != nil { - return nil, nil, err + return nil, err } var nd ipld.Node if adder.Trickle { @@ -126,15 +129,10 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error) nd, err = balanced.Layout(db) } if err != nil { - return nil, nil, err + return nil, err } - chunking, err := extractChunkingManifest(adder.ctx, adder.bufferedDS, nd.Cid()) - if err != nil { - return nil, nil, err - } - - return nd, chunking, adder.bufferedDS.Commit() + return nd, adder.bufferedDS.Commit() } // RootNode returns the mfs root node @@ -401,13 +399,41 @@ func (adder *Adder) addFile(path string, file files.File) error { } } - dagnode, chunkingManifest, err := adder.add(reader) + dagnode, err := adder.add(reader) if err != nil { return err } - // TODO: place the chunkingManifest into the "big file store". - // The big file store should be a field of the adder. - _ = chunkingManifest + + if adder.RawFileHash != nil { + chunkingManifest, err := extractChunkingManifest(adder.ctx, adder.bufferedDS, dagnode.Cid()) + if err != nil { + return err + } + + // Make more generic + if *adder.RawFileHash == multihash.SHA2_256 { + hasher := sha256.New() + for _, c := range chunkingManifest.Chunks { + nd, err := adder.bufferedDS.Get(adder.ctx, c.ChunkCid) + if err != nil { + return err + } + + if _, err := hasher.Write(nd.RawData()); err != nil { + return err + } + } + mh, err := multihash.Encode(hasher.Sum(nil), multihash.SHA2_256) + if err != nil { + return err + } + chunkingManifest.StreamCid = cid.NewCidV1(cid.Raw, mh) + } + + // TODO: place the chunkingManifest into the "big file store". + // The big file store should be a field of the adder. + _ = chunkingManifest + } // patch it into the root return adder.addNode(dagnode, path) diff --git a/go.mod b/go.mod index 25e25fcd3..8141a1029 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 - github.com/ipfs/go-bitswap v0.3.3 + github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.4 github.com/ipfs/go-cid v0.0.7 @@ -55,7 +55,7 @@ require ( github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 - github.com/ipfs/interface-go-ipfs-core v0.4.0 + github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59 github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c diff --git a/go.sum b/go.sum index dcb0966c8..4db05f15a 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSA github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= -github.com/ipfs/go-bitswap v0.3.3 h1:CrTO3OiOYFBcdliw074/C7T2QYHEOsPClgvR6RIYcO4= -github.com/ipfs/go-bitswap v0.3.3/go.mod h1:AyWWfN3moBzQX0banEtfKOfbXb3ZeoOeXnZGNPV9S6w= +github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120 h1:EH6dTmz867wv1V2rwPyOsQ4SDRxyGbikX/PTlvMPBsM= +github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120/go.mod h1:fjCW0ajzfqPml6HQGSiR+8NVwFPpgIItCyjG70BvCV0= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -416,6 +416,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= +github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59 h1:qR12199H3ZtmgCWLTftgwvojNU3xwRkpL7KHOa8jFPo= +github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc h1:BdI33Q56hLWG9Ef0WbQ7z+dwmbRYhTb45SMjw0RudbQ= github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc/go.mod h1:WdIgzcEjFqydQ7jH+BXzGYxVCmLeAs5nP8Vu3Rege2Y= github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= @@ -722,6 +724,8 @@ github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAo github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= github.com/libp2p/go-sockaddr v0.0.2 h1:tCuXfpA9rq7llM/v834RKc/Xvovy/AqM9kHvTV/jY/Q= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= +github.com/libp2p/go-sockaddr v0.1.0 h1:Y4s3/jNoryVRKEBrkJ576F17CPOaMIzUeCsg7dlTDj0= +github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-socket-activation v0.0.2 h1:VLU3IbrUUqu4DMhxA9857Q63qUpEAbCz5RqSnLCx5jE= github.com/libp2p/go-socket-activation v0.0.2/go.mod h1:KP44C+yZ7gA8sTxavgaD0b8vXVFJwam2CEW0s7+f094= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= diff --git a/namesys/dns.go b/namesys/dns.go index 0b48ad34b..738612f46 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -3,9 +3,9 @@ package namesys import ( "context" "errors" + "fmt" "net" "strings" - "fmt" path "github.com/ipfs/go-path" opts "github.com/ipfs/interface-go-ipfs-core/options/namesys"