mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-27 13:27:50 +08:00
add largeblocks option to ipfs add. calculate streamCid for SHA2_256
This commit is contained in:
Adin Schmahmann
parent
f9f337de20
commit
6be999ea0a
@ -45,6 +45,7 @@ const (
|
||||
hashOptionName = "hash"
|
||||
inlineOptionName = "inline"
|
||||
inlineLimitOptionName = "inline-limit"
|
||||
rawFileHashName = "raw-file-hash"
|
||||
)
|
||||
|
||||
const adderOutChanSize = 8
|
||||
@ -140,6 +141,7 @@ only-hash, and progress/status related flags) will change the final hash.
|
||||
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
|
||||
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
|
||||
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
|
||||
cmds.StringOption(rawFileHashName, "Additionally reference and share files using the hash of the full file. Takes the name of a hash function (experimental)"),
|
||||
},
|
||||
PreRun: func(req *cmds.Request, env cmds.Environment) error {
|
||||
quiet, _ := req.Options[quietOptionName].(bool)
|
||||
@ -180,6 +182,7 @@ only-hash, and progress/status related flags) will change the final hash.
|
||||
hashFunStr, _ := req.Options[hashOptionName].(string)
|
||||
inline, _ := req.Options[inlineOptionName].(bool)
|
||||
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
|
||||
rawFileHashStr, rawFileSet := req.Options[rawFileHashName].(string)
|
||||
|
||||
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
|
||||
if !ok {
|
||||
@ -227,6 +230,15 @@ only-hash, and progress/status related flags) will change the final hash.
|
||||
opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
|
||||
}
|
||||
|
||||
if rawFileSet {
|
||||
var rawHashFunCode uint64
|
||||
rawHashFunCode, ok = mh.Names[strings.ToLower(rawFileHashStr)]
|
||||
if !ok {
|
||||
return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(rawFileHashStr))
|
||||
}
|
||||
opts = append(opts, options.Unixfs.RawFileHash(rawHashFunCode))
|
||||
}
|
||||
|
||||
opts = append(opts, nil) // events option placeholder
|
||||
|
||||
var added int
|
||||
|
||||
@ -151,6 +151,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
|
||||
}
|
||||
}
|
||||
|
||||
if settings.RawFileHashSet {
|
||||
fileAdder.RawFileHash = &settings.RawFileHash
|
||||
}
|
||||
|
||||
if settings.OnlyHash {
|
||||
md := dagtest.Mock()
|
||||
emptyDirNode := ft.EmptyDirNode()
|
||||
|
||||
@ -2,8 +2,10 @@ package coreunix
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"io"
|
||||
gopath "path"
|
||||
"strconv"
|
||||
@ -61,24 +63,25 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG
|
||||
|
||||
// Adder holds the switches passed to the `add` command.
|
||||
type Adder struct {
|
||||
ctx context.Context
|
||||
pinning pin.Pinner
|
||||
gcLocker bstore.GCLocker
|
||||
dagService ipld.DAGService
|
||||
bufferedDS *ipld.BufferedDAG
|
||||
Out chan<- interface{}
|
||||
Progress bool
|
||||
Pin bool
|
||||
Trickle bool
|
||||
RawLeaves bool
|
||||
Silent bool
|
||||
NoCopy bool
|
||||
Chunker string
|
||||
mroot *mfs.Root
|
||||
unlocker bstore.Unlocker
|
||||
tempRoot cid.Cid
|
||||
CidBuilder cid.Builder
|
||||
liveNodes uint64
|
||||
ctx context.Context
|
||||
pinning pin.Pinner
|
||||
gcLocker bstore.GCLocker
|
||||
dagService ipld.DAGService
|
||||
bufferedDS *ipld.BufferedDAG
|
||||
Out chan<- interface{}
|
||||
Progress bool
|
||||
Pin bool
|
||||
Trickle bool
|
||||
RawLeaves bool
|
||||
Silent bool
|
||||
NoCopy bool
|
||||
RawFileHash *uint64
|
||||
Chunker string
|
||||
mroot *mfs.Root
|
||||
unlocker bstore.Unlocker
|
||||
tempRoot cid.Cid
|
||||
CidBuilder cid.Builder
|
||||
liveNodes uint64
|
||||
}
|
||||
|
||||
func (adder *Adder) mfsRoot() (*mfs.Root, error) {
|
||||
@ -101,10 +104,10 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) {
|
||||
}
|
||||
|
||||
// Constructs a node from reader's data, and adds it. Doesn't pin.
|
||||
func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error) {
|
||||
func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
|
||||
chnk, err := chunker.FromString(reader, adder.Chunker)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params := ihelper.DagBuilderParams{
|
||||
@ -117,7 +120,7 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error)
|
||||
|
||||
db, err := params.New(chnk)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
var nd ipld.Node
|
||||
if adder.Trickle {
|
||||
@ -126,15 +129,10 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, *ChunkingManifest, error)
|
||||
nd, err = balanced.Layout(db)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chunking, err := extractChunkingManifest(adder.ctx, adder.bufferedDS, nd.Cid())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return nd, chunking, adder.bufferedDS.Commit()
|
||||
return nd, adder.bufferedDS.Commit()
|
||||
}
|
||||
|
||||
// RootNode returns the mfs root node
|
||||
@ -401,13 +399,41 @@ func (adder *Adder) addFile(path string, file files.File) error {
|
||||
}
|
||||
}
|
||||
|
||||
dagnode, chunkingManifest, err := adder.add(reader)
|
||||
dagnode, err := adder.add(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: place the chunkingManifest into the "big file store".
|
||||
// The big file store should be a field of the adder.
|
||||
_ = chunkingManifest
|
||||
|
||||
if adder.RawFileHash != nil {
|
||||
chunkingManifest, err := extractChunkingManifest(adder.ctx, adder.bufferedDS, dagnode.Cid())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make more generic
|
||||
if *adder.RawFileHash == multihash.SHA2_256 {
|
||||
hasher := sha256.New()
|
||||
for _, c := range chunkingManifest.Chunks {
|
||||
nd, err := adder.bufferedDS.Get(adder.ctx, c.ChunkCid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := hasher.Write(nd.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mh, err := multihash.Encode(hasher.Sum(nil), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkingManifest.StreamCid = cid.NewCidV1(cid.Raw, mh)
|
||||
}
|
||||
|
||||
// TODO: place the chunkingManifest into the "big file store".
|
||||
// The big file store should be a field of the adder.
|
||||
_ = chunkingManifest
|
||||
}
|
||||
|
||||
// patch it into the root
|
||||
return adder.addNode(dagnode, path)
|
||||
|
||||
4
go.mod
4
go.mod
@ -15,7 +15,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/hashicorp/go-multierror v1.1.0
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/ipfs/go-bitswap v0.3.3
|
||||
github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120
|
||||
github.com/ipfs/go-block-format v0.0.2
|
||||
github.com/ipfs/go-blockservice v0.1.4
|
||||
github.com/ipfs/go-cid v0.0.7
|
||||
@ -55,7 +55,7 @@ require (
|
||||
github.com/ipfs/go-pinning-service-http-client v0.1.0
|
||||
github.com/ipfs/go-unixfs v0.2.4
|
||||
github.com/ipfs/go-verifcid v0.0.1
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.0
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59
|
||||
github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc
|
||||
github.com/jbenet/go-is-domain v1.0.5
|
||||
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
|
||||
|
||||
8
go.sum
8
go.sum
@ -263,8 +263,8 @@ github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSA
|
||||
github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
|
||||
github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps=
|
||||
github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
|
||||
github.com/ipfs/go-bitswap v0.3.3 h1:CrTO3OiOYFBcdliw074/C7T2QYHEOsPClgvR6RIYcO4=
|
||||
github.com/ipfs/go-bitswap v0.3.3/go.mod h1:AyWWfN3moBzQX0banEtfKOfbXb3ZeoOeXnZGNPV9S6w=
|
||||
github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120 h1:EH6dTmz867wv1V2rwPyOsQ4SDRxyGbikX/PTlvMPBsM=
|
||||
github.com/ipfs/go-bitswap v0.3.4-0.20210203213610-ec4d6a9f0120/go.mod h1:fjCW0ajzfqPml6HQGSiR+8NVwFPpgIItCyjG70BvCV0=
|
||||
github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
|
||||
github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
@ -416,6 +416,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2
|
||||
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59 h1:qR12199H3ZtmgCWLTftgwvojNU3xwRkpL7KHOa8jFPo=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210203221434-f39297b7ac59/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
|
||||
github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc h1:BdI33Q56hLWG9Ef0WbQ7z+dwmbRYhTb45SMjw0RudbQ=
|
||||
github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc/go.mod h1:WdIgzcEjFqydQ7jH+BXzGYxVCmLeAs5nP8Vu3Rege2Y=
|
||||
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
|
||||
@ -722,6 +724,8 @@ github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAo
|
||||
github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw=
|
||||
github.com/libp2p/go-sockaddr v0.0.2 h1:tCuXfpA9rq7llM/v834RKc/Xvovy/AqM9kHvTV/jY/Q=
|
||||
github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
|
||||
github.com/libp2p/go-sockaddr v0.1.0 h1:Y4s3/jNoryVRKEBrkJ576F17CPOaMIzUeCsg7dlTDj0=
|
||||
github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
|
||||
github.com/libp2p/go-socket-activation v0.0.2 h1:VLU3IbrUUqu4DMhxA9857Q63qUpEAbCz5RqSnLCx5jE=
|
||||
github.com/libp2p/go-socket-activation v0.0.2/go.mod h1:KP44C+yZ7gA8sTxavgaD0b8vXVFJwam2CEW0s7+f094=
|
||||
github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
|
||||
|
||||
@ -3,9 +3,9 @@ package namesys
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"fmt"
|
||||
|
||||
path "github.com/ipfs/go-path"
|
||||
opts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user