package helpers import ( "io" "os" "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" files "gx/ipfs/QmUyfy4QSr3NXym4etEiRyxBLqqAeKHJuRdi8AACxg63fZ/go-ipfs-cmdkit/files" ) // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { dserv dag.DAGService spl chunk.Splitter recvdErr error rawLeaves bool nextData []byte // the next item to return. maxlinks int batch *dag.Batch fullPath string stat os.FileInfo prefix *cid.Prefix } type DagBuilderParams struct { // Maximum number of links per intermediate node Maxlinks int // RawLeaves signifies that the importer should use raw ipld nodes as leaves // instead of using the unixfs TRaw type RawLeaves bool // CID Prefix to use if set Prefix *cid.Prefix // DAGService to write blocks to (required) Dagserv dag.DAGService // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool } // Generate a new DagBuilderHelper from the given params, which data source comes // from chunks object func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper { db := &DagBuilderHelper{ dserv: dbp.Dagserv, spl: spl, rawLeaves: dbp.RawLeaves, prefix: dbp.Prefix, maxlinks: dbp.Maxlinks, batch: dbp.Dagserv.Batch(), } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() db.stat = fi.Stat() } return db } // prepareNext consumes the next item from the splitter and puts it // in the nextData field. it is idempotent-- if nextData is full // it will do nothing. func (db *DagBuilderHelper) prepareNext() { // if we already have data waiting to be consumed, we're ready if db.nextData != nil || db.recvdErr != nil { return } db.nextData, db.recvdErr = db.spl.NextBytes() if db.recvdErr == io.EOF { db.recvdErr = nil } } // Done returns whether or not we're done consuming the incoming data. func (db *DagBuilderHelper) Done() bool { // ensure we have an accurate perspective on data // as `done` this may be called before `next`. db.prepareNext() // idempotent if db.recvdErr != nil { return false } return db.nextData == nil } // Next returns the next chunk of data to be inserted into the dag // if it returns nil, that signifies that the stream is at an end, and // that the current building operation should finish func (db *DagBuilderHelper) Next() ([]byte, error) { db.prepareNext() // idempotent d := db.nextData db.nextData = nil // signal we've consumed it if db.recvdErr != nil { return nil, db.recvdErr } else { return d, nil } } // GetDagServ returns the dagservice object this Helper is using func (db *DagBuilderHelper) GetDagServ() dag.DAGService { return db.dserv } // NewUnixfsNode creates a new Unixfs node to represent a file. func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { n := &UnixfsNode{ node: new(dag.ProtoNode), ufmt: &ft.FSNode{Type: ft.TFile}, } n.SetPrefix(db.prefix) return n } // newUnixfsBlock creates a new Unixfs node to represent a raw data block func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode { n := &UnixfsNode{ node: new(dag.ProtoNode), ufmt: &ft.FSNode{Type: ft.TRaw}, } n.SetPrefix(db.prefix) return n } // FillNodeLayer will add datanodes as children to the give node until // at most db.indirSize ndoes are added // func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error { // while we have room AND we're not done for node.NumChildren() < db.maxlinks && !db.Done() { child, err := db.GetNextDataNode() if err != nil { return err } if err := node.AddChild(child, db); err != nil { return err } } return nil } func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { data, err := db.Next() if err != nil { return nil, err } if data == nil { // we're done! return nil, nil } if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded } if db.rawLeaves { if db.prefix == nil { return &UnixfsNode{ rawnode: dag.NewRawNode(data), raw: true, }, nil } else { rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) if err != nil { return nil, err } return &UnixfsNode{ rawnode: rawnode, raw: true, }, nil } } else { blk := db.newUnixfsBlock() blk.SetData(data) return blk, nil } } func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) { if db.fullPath != "" { node.SetPosInfo(offset, db.fullPath, db.stat) } } func (db *DagBuilderHelper) Add(node *UnixfsNode) (node.Node, error) { dn, err := node.GetDagNode() if err != nil { return nil, err } _, err = db.dserv.Add(dn) if err != nil { return nil, err } return dn, nil } func (db *DagBuilderHelper) Maxlinks() int { return db.maxlinks } func (db *DagBuilderHelper) Close() error { return db.batch.Commit() }