From 16a54a85615267dae18e49f83ebf79dbf6e80556 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 12:05:41 +0100 Subject: [PATCH 01/10] Golint: unixfs main module License: MIT Signed-off-by: Hector Sanjuan --- unixfs/{format.go => unixfs.go} | 41 ++++++++++++++++++----- unixfs/{format_test.go => unixfs_test.go} | 0 2 files changed, 33 insertions(+), 8 deletions(-) rename unixfs/{format.go => unixfs.go} (74%) rename unixfs/{format_test.go => unixfs_test.go} (100%) diff --git a/unixfs/format.go b/unixfs/unixfs.go similarity index 74% rename from unixfs/format.go rename to unixfs/unixfs.go index 26d4b0cc3..d04a461ed 100644 --- a/unixfs/format.go +++ b/unixfs/unixfs.go @@ -1,4 +1,4 @@ -// Package format implements a data format for files in the IPFS filesystem It +// Package unixfs implements a data format for files in the IPFS filesystem It // is not the only format in ipfs, but it is the one that the filesystem // assumes package unixfs @@ -6,11 +6,13 @@ package unixfs import ( "errors" + proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" + dag "github.com/ipfs/go-ipfs/merkledag" pb "github.com/ipfs/go-ipfs/unixfs/pb" - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" ) +// Shorthands for protobuffer types const ( TRaw = pb.Data_Raw TFile = pb.Data_File @@ -20,10 +22,14 @@ const ( THAMTShard = pb.Data_HAMTShard ) -var ErrMalformedFileFormat = errors.New("malformed data in file format") -var ErrInvalidDirLocation = errors.New("found directory node in unexpected place") -var ErrUnrecognizedType = errors.New("unrecognized node type") +// Common errors +var ( + ErrMalformedFileFormat = errors.New("malformed data in file format") + ErrInvalidDirLocation = errors.New("found directory node in unexpected place") + ErrUnrecognizedType = errors.New("unrecognized node type") +) +// FromBytes unmarshals a byte slice as protobuf Data. func FromBytes(data []byte) (*pb.Data, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) @@ -33,6 +39,8 @@ func FromBytes(data []byte) (*pb.Data, error) { return pbdata, nil } +// FilePBData creates a protobuf File with the given +// byte slice and returns the marshaled protobuf bytes representing it. func FilePBData(data []byte, totalsize uint64) []byte { pbfile := new(pb.Data) typ := pb.Data_File @@ -98,6 +106,7 @@ func SymlinkData(path string) ([]byte, error) { return out, nil } +// UnwrapData unmarshals a protobuf messages and returns the contents. func UnwrapData(data []byte) ([]byte, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) @@ -107,6 +116,10 @@ func UnwrapData(data []byte) ([]byte, error) { return pbdata.GetData(), nil } +// DataSize returns the size of the contents in protobuf wrapped slice. +// For raw data it simply provides the length of it. For Data_Files, it +// will return the associated filesize. Note that Data_Directories will +// return an error. func DataSize(data []byte) (uint64, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) @@ -116,16 +129,17 @@ func DataSize(data []byte) (uint64, error) { switch pbdata.GetType() { case pb.Data_Directory: - return 0, errors.New("Cant get data size of directory!") + return 0, errors.New("Cant get data size of directory") case pb.Data_File: return pbdata.GetFilesize(), nil case pb.Data_Raw: return uint64(len(pbdata.GetData())), nil default: - return 0, errors.New("Unrecognized node data type!") + return 0, errors.New("Unrecognized node data type") } } +// An FSNode represents a filesystem object. type FSNode struct { Data []byte @@ -139,6 +153,7 @@ type FSNode struct { Type pb.Data_DataType } +// FSNodeFromBytes unmarshal a protobuf message onto an FSNode. func FSNodeFromBytes(b []byte) (*FSNode, error) { pbn := new(pb.Data) err := proto.Unmarshal(b, pbn) @@ -160,11 +175,13 @@ func (n *FSNode) AddBlockSize(s uint64) { n.blocksizes = append(n.blocksizes, s) } +// RemoveBlockSize removes the given child block's size. func (n *FSNode) RemoveBlockSize(i int) { n.subtotal -= n.blocksizes[i] n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...) } +// GetBytes marshals this node as a protobuf message. func (n *FSNode) GetBytes() ([]byte, error) { pbn := new(pb.Data) pbn.Type = &n.Type @@ -180,16 +197,19 @@ func (n *FSNode) FileSize() uint64 { return uint64(len(n.Data)) + n.subtotal } +// NumChildren returns the number of child blocks of this node func (n *FSNode) NumChildren() int { return len(n.blocksizes) } +// Metadata is used to store additional FSNode information. type Metadata struct { MimeType string Size uint64 } -//MetadataFromBytes Unmarshals a protobuf message into Metadata. +// MetadataFromBytes Unmarshals a protobuf Data message into Metadata. +// The provided slice should have been encoded with BytesForMetadata(). func MetadataFromBytes(b []byte) (*Metadata, error) { pbd := new(pb.Data) err := proto.Unmarshal(b, pbd) @@ -210,12 +230,16 @@ func MetadataFromBytes(b []byte) (*Metadata, error) { return md, nil } +// Bytes marshals Metadata as a protobuf message of Metadata type. func (m *Metadata) Bytes() ([]byte, error) { pbm := new(pb.Metadata) pbm.MimeType = &m.MimeType return proto.Marshal(pbm) } +// BytesForMetadata wraps the given Metadata as a profobuf message of Data type, +// setting the DataType to Metadata. The wrapped bytes are itself the +// result of calling m.Bytes(). func BytesForMetadata(m *Metadata) ([]byte, error) { pbd := new(pb.Data) pbd.Filesize = proto.Uint64(m.Size) @@ -230,6 +254,7 @@ func BytesForMetadata(m *Metadata) ([]byte, error) { return proto.Marshal(pbd) } +// EmptyDirNode creates an empty folder Protonode. func EmptyDirNode() *dag.ProtoNode { return dag.NodeWithData(FolderPBData()) } diff --git a/unixfs/format_test.go b/unixfs/unixfs_test.go similarity index 100% rename from unixfs/format_test.go rename to unixfs/unixfs_test.go From 694ab67952d7668df59a5f6109b80aade10bf023 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 12:43:13 +0100 Subject: [PATCH 02/10] Golint: unixfs/archive License: MIT Signed-off-by: Hector Sanjuan --- unixfs/archive/archive.go | 1 + unixfs/archive/tar/writer.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/unixfs/archive/archive.go b/unixfs/archive/archive.go index 72b3e2dac..7a561992e 100644 --- a/unixfs/archive/archive.go +++ b/unixfs/archive/archive.go @@ -1,3 +1,4 @@ +// Package archive provides utilities to archive and compress a [Unixfs] DAG. package archive import ( diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 17614e10e..4503f5593 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -1,3 +1,5 @@ +// Package tar provides functionality to write a unixfs merkledag +// as a tar archive. package tar import ( @@ -69,6 +71,7 @@ func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error return nil } +// WriteNode adds a node to the archive. func (w *Writer) WriteNode(nd ipld.Node, fpath string) error { switch nd := nd.(type) { case *mdag.ProtoNode: @@ -106,6 +109,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error { } } +// Close closes the tar writer. func (w *Writer) Close() error { return w.TarW.Close() } From 16934b9c68bad72dec2b0102c9c9847bf5f98728 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 12:43:30 +0100 Subject: [PATCH 03/10] Golint: unixfs/hamt Note, stuttering required renaming of HamtShard to Shard. License: MIT Signed-off-by: Hector Sanjuan --- unixfs/hamt/hamt.go | 70 +++++++++++++++++---------------- unixfs/hamt/hamt_stress_test.go | 12 +++--- unixfs/hamt/hamt_test.go | 24 +++++------ 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index 80d97d8ec..70c0b371c 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -38,10 +38,12 @@ import ( ) const ( + // HashMurmur3 is the multiformats identifier for Murmur3 HashMurmur3 uint64 = 0x22 ) -type HamtShard struct { +// A Shard represents the HAMT. It should be initialized with NewShard(). +type Shard struct { nd *dag.ProtoNode bitfield *big.Int @@ -66,9 +68,9 @@ type child interface { Label() string } -// NewHamtShard creates a new, empty HAMT shard with the given size. -func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) { - ds, err := makeHamtShard(dserv, size) +// NewShard creates a new, empty HAMT shard with the given size. +func NewShard(dserv ipld.DAGService, size int) (*Shard, error) { + ds, err := makeShard(dserv, size) if err != nil { return nil, err } @@ -79,13 +81,13 @@ func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) { return ds, nil } -func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) { +func makeShard(ds ipld.DAGService, size int) (*Shard, error) { lg2s := int(math.Log2(float64(size))) if 1<= len(ds.children) || i < 0 { return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)") } @@ -281,7 +283,7 @@ func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) { // loadChild reads the i'th child node of this shard from disk and returns it // as a 'child' interface -func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) { +func (ds *Shard) loadChild(ctx context.Context, i int) (child, error) { lnk := ds.nd.Links()[i] if len(lnk.Name) < ds.maxpadlen { return nil, fmt.Errorf("invalid link name '%s'", lnk.Name) @@ -326,12 +328,12 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) { return c, nil } -func (ds *HamtShard) setChild(i int, c child) { +func (ds *Shard) setChild(i int, c child) { ds.children[i] = c } // Link returns a merklelink to this shard node -func (ds *HamtShard) Link() (*ipld.Link, error) { +func (ds *Shard) Link() (*ipld.Link, error) { nd, err := ds.Node() if err != nil { return nil, err @@ -345,7 +347,7 @@ func (ds *HamtShard) Link() (*ipld.Link, error) { return ipld.MakeLink(nd) } -func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error { +func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error { if lnk == nil { return os.ErrNotExist } @@ -364,7 +366,7 @@ func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error { return nil } -func (ds *HamtShard) rmChild(i int) error { +func (ds *Shard) rmChild(i int) error { if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) { return fmt.Errorf("hamt: attempted to remove child with out of range index") } @@ -378,7 +380,7 @@ func (ds *HamtShard) rmChild(i int) error { return nil } -func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error { +func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error { idx := hv.Next(ds.tableSizeLg2) if ds.bitfield.Bit(int(idx)) == 1 { cindex := ds.indexForBitPos(idx) @@ -389,7 +391,7 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb } switch child := child.(type) { - case *HamtShard: + case *Shard: return child.getValue(ctx, hv, key, cb) case *shardValue: if child.key == key { @@ -401,7 +403,8 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb return os.ErrNotExist } -func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) { +// EnumLinks collects all links in the Shard. +func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) { var links []*ipld.Link err := ds.ForEachLink(ctx, func(l *ipld.Link) error { links = append(links, l) @@ -410,7 +413,8 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) { return links, err } -func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { +// ForEachLink walks the Shard and calls the given function. +func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { return ds.walkTrie(ctx, func(sv *shardValue) error { lnk := sv.val lnk.Name = sv.key @@ -419,7 +423,7 @@ func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) }) } -func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) error { +func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error { for i := 0; i < ds.tableSize; i++ { if ds.bitfield.Bit(i) == 0 { continue @@ -440,7 +444,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e return err } - case *HamtShard: + case *Shard: err := c.walkTrie(ctx, cb) if err != nil { return err @@ -452,7 +456,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e return nil } -func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error { +func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error { idx := hv.Next(ds.tableSizeLg2) if ds.bitfield.Bit(idx) != 1 { @@ -467,7 +471,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, } switch child := child.(type) { - case *HamtShard: + case *Shard: err := child.modifyValue(ctx, hv, key, val) if err != nil { return err @@ -510,7 +514,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, } // replace value with another shard, one level deeper - ns, err := NewHamtShard(ds.dserv, ds.tableSize) + ns, err := NewShard(ds.dserv, ds.tableSize) if err != nil { return err } @@ -540,7 +544,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, // indexForBitPos returns the index within the collapsed array corresponding to // the given bit in the bitset. The collapsed array contains only one entry // per bit set in the bitfield, and this function is used to map the indices. -func (ds *HamtShard) indexForBitPos(bp int) int { +func (ds *Shard) indexForBitPos(bp int) int { // TODO: an optimization could reuse the same 'mask' here and change the size // as needed. This isnt yet done as the bitset package doesnt make it easy // to do. @@ -553,6 +557,6 @@ func (ds *HamtShard) indexForBitPos(bp int) int { } // linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix -func (ds *HamtShard) linkNamePrefix(idx int) string { +func (ds *Shard) linkNamePrefix(idx int) string { return fmt.Sprintf(ds.prefixPadStr, idx) } diff --git a/unixfs/hamt/hamt_stress_test.go b/unixfs/hamt/hamt_stress_test.go index e746a44b5..185e385e1 100644 --- a/unixfs/hamt/hamt_stress_test.go +++ b/unixfs/hamt/hamt_stress_test.go @@ -94,7 +94,7 @@ func TestOrderConsistency(t *testing.T) { } } -func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) error { +func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error { ctx := context.TODO() for _, n := range keep { _, err := s.Find(ctx, n) @@ -113,9 +113,9 @@ func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) er return nil } -func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*HamtShard, error) { +func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) { ctx := context.TODO() - s, err := NewHamtShard(ds, width) + s, err := NewShard(ds, width) if err != nil { return nil, err } @@ -189,9 +189,9 @@ func genOpSet(seed int64, keep, temp []string) []testOp { } // executes the given op set with a repl to allow easier debugging -/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*HamtShard, error) { +/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) { - s, err := NewHamtShard(ds, width) + s, err := NewShard(ds, width) if err != nil { return nil, err } @@ -244,7 +244,7 @@ mainloop: } case "restart": var err error - s, err = NewHamtShard(ds, width) + s, err = NewShard(ds, width) if err != nil { panic(err) } diff --git a/unixfs/hamt/hamt_test.go b/unixfs/hamt/hamt_test.go index f89e9ac57..72f74526a 100644 --- a/unixfs/hamt/hamt_test.go +++ b/unixfs/hamt/hamt_test.go @@ -26,14 +26,14 @@ func shuffle(seed int64, arr []string) { } } -func makeDir(ds ipld.DAGService, size int) ([]string, *HamtShard, error) { +func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) { return makeDirWidth(ds, size, 256) } -func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, error) { +func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) { ctx := context.Background() - s, _ := NewHamtShard(ds, width) + s, _ := NewShard(ds, width) var dirs []string for i := 0; i < size; i++ { @@ -54,7 +54,7 @@ func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, er return dirs, s, nil } -func assertLink(s *HamtShard, name string, found bool) error { +func assertLink(s *Shard, name string, found bool) error { _, err := s.Find(context.Background(), name) switch err { case os.ErrNotExist: @@ -74,7 +74,7 @@ func assertLink(s *HamtShard, name string, found bool) error { } } -func assertSerializationWorks(ds ipld.DAGService, s *HamtShard) error { +func assertSerializationWorks(ds ipld.DAGService, s *Shard) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() nd, err := s.Node() @@ -141,7 +141,7 @@ func TestBasicSet(t *testing.T) { func TestDirBuilding(t *testing.T) { ds := mdtest.Mock() - _, _ = NewHamtShard(ds, 256) + _, _ = NewShard(ds, 256) _, s, err := makeDir(ds, 200) if err != nil { @@ -164,7 +164,7 @@ func TestDirBuilding(t *testing.T) { func TestShardReload(t *testing.T) { ds := mdtest.Mock() - _, _ = NewHamtShard(ds, 256) + _, _ = NewShard(ds, 256) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -307,7 +307,7 @@ func TestSetAfterMarshal(t *testing.T) { func TestDuplicateAddShard(t *testing.T) { ds := mdtest.Mock() - dir, _ := NewHamtShard(ds, 256) + dir, _ := NewShard(ds, 256) nd := new(dag.ProtoNode) ctx := context.Background() @@ -430,7 +430,7 @@ func TestRemoveElemsAfterMarshal(t *testing.T) { func TestBitfieldIndexing(t *testing.T) { ds := mdtest.Mock() - s, _ := NewHamtShard(ds, 256) + s, _ := NewShard(ds, 256) set := func(i int) { s.bitfield.SetBit(s.bitfield, i, 1) @@ -466,7 +466,7 @@ func TestSetHamtChild(t *testing.T) { ctx := context.Background() ds := mdtest.Mock() - s, _ := NewHamtShard(ds, 256) + s, _ := NewShard(ds, 256) e := ft.EmptyDirNode() ds.Add(ctx, e) @@ -527,7 +527,7 @@ func BenchmarkHAMTSet(b *testing.B) { ctx := context.Background() ds := mdtest.Mock() - sh, _ := NewHamtShard(ds, 256) + sh, _ := NewShard(ds, 256) nd, err := sh.Node() if err != nil { b.Fatal(err) @@ -560,7 +560,7 @@ func BenchmarkHAMTSet(b *testing.B) { } func TestHamtBadSize(t *testing.T) { - _, err := NewHamtShard(nil, 7) + _, err := NewShard(nil, 7) if err == nil { t.Fatal("should have failed to construct hamt with bad size") } From f074292e4802d3f0ce4615944701568ab1253739 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 13:08:58 +0100 Subject: [PATCH 04/10] Golint: unixfs/mod unixfs/test License: MIT Signed-off-by: Hector Sanjuan --- unixfs/mod/dagmodifier.go | 18 +++++++++++++----- unixfs/test/utils.go | 13 +++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index 8bf9382dd..05c5c3587 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -1,3 +1,5 @@ +// Package mod provides DAG modification utilities to, for example, +// insert additional nodes in a unixfs DAG or truncate them. package mod import ( @@ -19,8 +21,12 @@ import ( ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) -var ErrSeekFail = errors.New("failed to seek properly") -var ErrUnrecognizedWhence = errors.New("unrecognized whence") +// Common errors +var ( + ErrSeekFail = errors.New("failed to seek properly") + ErrUnrecognizedWhence = errors.New("unrecognized whence") + ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)") +) // 2MB var writebufferSize = 1 << 21 @@ -46,8 +52,6 @@ type DagModifier struct { read uio.DagReader } -var ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)") - // NewDagModifier returns a new DagModifier, the Cid prefix for newly // created nodes will be inherted from the passed in node. If the Cid // version if not 0 raw leaves will also be enabled. The Prefix and @@ -412,7 +416,7 @@ func (dm *DagModifier) readPrep() error { return nil } -// Read data from this dag starting at the current offset +// CtxReadFull reads data from this dag starting at the current offset func (dm *DagModifier) CtxReadFull(ctx context.Context, b []byte) (int, error) { err := dm.readPrep() if err != nil { @@ -438,6 +442,8 @@ func (dm *DagModifier) HasChanges() bool { return dm.wrBuf != nil } +// Seek modifies the offset according to whence. See unixfs/io for valid whence +// values. func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) { err := dm.Sync() if err != nil { @@ -479,6 +485,8 @@ func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) { return int64(dm.curWrOff), nil } +// Truncate truncates the current Node to 'size' and replaces it with the +// new one. func (dm *DagModifier) Truncate(size int64) error { err := dm.Sync() if err != nil { diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go index 5e1977ddb..0ca47c842 100644 --- a/unixfs/test/utils.go +++ b/unixfs/test/utils.go @@ -21,6 +21,7 @@ import ( ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) +// SizeSplitterGen creates a generator. func SizeSplitterGen(size int64) chunk.SplitterGen { return func(r io.Reader) chunk.Splitter { return chunk.NewSizeSplitter(r, size) @@ -41,10 +42,13 @@ type NodeOpts struct { RawLeavesUsed bool } -var UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()} -var UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true} -var UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true} -var UseBlake2b256 NodeOpts +// Some shorthands for NodeOpts. +var ( + UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()} + UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true} + UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true} + UseBlake2b256 NodeOpts +) func init() { UseBlake2b256 = UseCidV1 @@ -88,6 +92,7 @@ func GetRandomNode(t testing.TB, dserv ipld.DAGService, size int64, opts NodeOpt return buf, node } +// ArrComp checks if two byte slices are the same. func ArrComp(a, b []byte) error { if len(a) != len(b) { return fmt.Errorf("Arrays differ in length. %d != %d", len(a), len(b)) From ca4271b7827fe3234d192011dac52e0b0b600e9d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 13:48:35 +0100 Subject: [PATCH 05/10] Golint: unixfs/io License: MIT Signed-off-by: Hector Sanjuan --- unixfs/io/bufdagreader.go | 4 +++- unixfs/io/dagreader.go | 14 ++++++++---- unixfs/io/dagreader_test.go | 2 +- unixfs/io/dirbuilder.go | 17 +++++++++++---- unixfs/io/doc.go | 2 +- unixfs/io/pbdagreader.go | 43 ++++++++++++++++++++----------------- 6 files changed, 51 insertions(+), 31 deletions(-) diff --git a/unixfs/io/bufdagreader.go b/unixfs/io/bufdagreader.go index 2f7358640..8c6fd0cf8 100644 --- a/unixfs/io/bufdagreader.go +++ b/unixfs/io/bufdagreader.go @@ -10,7 +10,9 @@ type bufDagReader struct { *bytes.Reader } -func NewBufDagReader(b []byte) *bufDagReader { +// newBufDagReader returns a DAG reader for the given byte slice. +// BufDagReader is used to read RawNodes. +func newBufDagReader(b []byte) *bufDagReader { return &bufDagReader{bytes.NewReader(b)} } diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index c6e9cf0d9..037e0a256 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -14,10 +14,15 @@ import ( ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) -var ErrIsDir = errors.New("this dag node is a directory") - -var ErrCantReadSymlinks = errors.New("cannot currently read symlinks") +// Common errors +var ( + ErrIsDir = errors.New("this dag node is a directory") + ErrCantReadSymlinks = errors.New("cannot currently read symlinks") +) +// A DagReader represents a ReadSeekCloser which offers additional methods +// like Size. Different implementations of readers are used for the different +// types of unixfs/protobuf-encoded nodes. type DagReader interface { ReadSeekCloser Size() uint64 @@ -25,6 +30,7 @@ type DagReader interface { Offset() int64 } +// A ReadSeekCloser implements interfaces to read, write, seek and close. type ReadSeekCloser interface { io.Reader io.Seeker @@ -37,7 +43,7 @@ type ReadSeekCloser interface { func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { switch n := n.(type) { case *mdag.RawNode: - return NewBufDagReader(n.RawData()), nil + return newBufDagReader(n.RawData()), nil case *mdag.ProtoNode: pb := new(ftpb.Data) if err := proto.Unmarshal(n.Data(), pb); err != nil { diff --git a/unixfs/io/dagreader_test.go b/unixfs/io/dagreader_test.go index 82ff10234..e3d3d042b 100644 --- a/unixfs/io/dagreader_test.go +++ b/unixfs/io/dagreader_test.go @@ -102,7 +102,7 @@ func TestSeekAndReadLarge(t *testing.T) { t.Fatal("seeked read failed") } - pbdr := reader.(*pbDagReader) + pbdr := reader.(*PBDagReader) var count int for i, p := range pbdr.promises { if i > 20 && i < 30 { diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index 2deb4bcc8..f2dee053c 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -8,8 +8,8 @@ import ( mdag "github.com/ipfs/go-ipfs/merkledag" format "github.com/ipfs/go-ipfs/unixfs" hamt "github.com/ipfs/go-ipfs/unixfs/hamt" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" + cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) @@ -25,11 +25,14 @@ var UseHAMTSharding = false // DefaultShardWidth is the default value used for hamt sharding width. var DefaultShardWidth = 256 +// Directory allows to work with UnixFS directory nodes, adding and removing +// children. It allows to work with different directory schemes, +// like the classic or the HAMT one. type Directory struct { dserv ipld.DAGService dirnode *mdag.ProtoNode - shard *hamt.HamtShard + shard *hamt.Shard } // NewDirectory returns a Directory. It needs a DAGService to add the Children @@ -37,7 +40,7 @@ func NewDirectory(dserv ipld.DAGService) *Directory { db := new(Directory) db.dserv = dserv if UseHAMTSharding { - s, err := hamt.NewHamtShard(dserv, DefaultShardWidth) + s, err := hamt.NewShard(dserv, DefaultShardWidth) if err != nil { panic(err) // will only panic if DefaultShardWidth is a bad value } @@ -113,7 +116,7 @@ func (d *Directory) AddChild(ctx context.Context, name string, nd ipld.Node) err } func (d *Directory) switchToSharding(ctx context.Context) error { - s, err := hamt.NewHamtShard(d.dserv, DefaultShardWidth) + s, err := hamt.NewShard(d.dserv, DefaultShardWidth) if err != nil { return err } @@ -136,6 +139,7 @@ func (d *Directory) switchToSharding(ctx context.Context) error { return nil } +// ForEachLink applies the given function to Links in the directory. func (d *Directory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { if d.shard == nil { for _, l := range d.dirnode.Links() { @@ -149,6 +153,7 @@ func (d *Directory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) e return d.shard.ForEachLink(ctx, f) } +// Links returns the all the links in the directory node. func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) { if d.shard == nil { return d.dirnode.Links(), nil @@ -157,6 +162,9 @@ func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) { return d.shard.EnumLinks(ctx) } +// Find returns the ipld.Node with the given name, if it is contained in this +// directory. Find only searches in the most inmediate links, and not +// recursively in the tree. func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) { if d.shard == nil { lnk, err := d.dirnode.GetNodeLink(name) @@ -179,6 +187,7 @@ func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) { return lnk.GetNode(ctx, d.dserv) } +// RemoveChild removes the child with the given name. func (d *Directory) RemoveChild(ctx context.Context, name string) error { if d.shard == nil { return d.dirnode.RemoveNodeLink(name) diff --git a/unixfs/io/doc.go b/unixfs/io/doc.go index b28755fc6..cf844bd23 100644 --- a/unixfs/io/doc.go +++ b/unixfs/io/doc.go @@ -1,3 +1,3 @@ -// package unixfs/io implements convenience objects for working with the ipfs +// Package io implements convenience objects for working with the ipfs // unixfs data format. package io diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 0c6bee832..4523481b3 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -15,8 +15,8 @@ import ( ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" ) -// DagReader provides a way to easily read the data contained in a dag. -type pbDagReader struct { +// PBDagReader provides a way to easily read the data contained in a dag. +type PBDagReader struct { serv ipld.NodeGetter // the node being read @@ -48,16 +48,16 @@ type pbDagReader struct { cancel func() } -var _ DagReader = (*pbDagReader)(nil) +var _ DagReader = (*PBDagReader)(nil) // NewPBFileReader constructs a new PBFileReader. -func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv ipld.NodeGetter) *pbDagReader { +func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv ipld.NodeGetter) *PBDagReader { fctx, cancel := context.WithCancel(ctx) curLinks := getLinkCids(n) - return &pbDagReader{ + return &PBDagReader{ node: n, serv: serv, - buf: NewBufDagReader(pb.GetData()), + buf: newBufDagReader(pb.GetData()), promises: make([]*ipld.NodePromise, len(curLinks)), links: curLinks, ctx: fctx, @@ -68,7 +68,7 @@ func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv const preloadSize = 10 -func (dr *pbDagReader) preloadNextNodes(ctx context.Context) { +func (dr *PBDagReader) preloadNextNodes(ctx context.Context) { beg := dr.linkPosition end := beg + preloadSize if end >= len(dr.links) { @@ -82,7 +82,7 @@ func (dr *pbDagReader) preloadNextNodes(ctx context.Context) { // precalcNextBuf follows the next link in line and loads it from the // DAGService, setting the next buffer to read from -func (dr *pbDagReader) precalcNextBuf(ctx context.Context) error { +func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error { if dr.buf != nil { dr.buf.Close() // Just to make sure dr.buf = nil @@ -119,7 +119,7 @@ func (dr *pbDagReader) precalcNextBuf(ctx context.Context) error { dr.buf = NewPBFileReader(dr.ctx, nxt, pb, dr.serv) return nil case ftpb.Data_Raw: - dr.buf = NewBufDagReader(pb.GetData()) + dr.buf = newBufDagReader(pb.GetData()) return nil case ftpb.Data_Metadata: return errors.New("shouldnt have had metadata object inside file") @@ -145,17 +145,17 @@ func getLinkCids(n ipld.Node) []*cid.Cid { } // Size return the total length of the data from the DAG structured file. -func (dr *pbDagReader) Size() uint64 { +func (dr *PBDagReader) Size() uint64 { return dr.pbdata.GetFilesize() } // Read reads data from the DAG structured file -func (dr *pbDagReader) Read(b []byte) (int, error) { +func (dr *PBDagReader) Read(b []byte) (int, error) { return dr.CtxReadFull(dr.ctx, b) } // CtxReadFull reads data from the DAG structured file -func (dr *pbDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) { +func (dr *PBDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) { if dr.buf == nil { if err := dr.precalcNextBuf(ctx); err != nil { return 0, err @@ -189,7 +189,8 @@ func (dr *pbDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) { } } -func (dr *pbDagReader) WriteTo(w io.Writer) (int64, error) { +// WriteTo writes to the given writer. +func (dr *PBDagReader) WriteTo(w io.Writer) (int64, error) { if dr.buf == nil { if err := dr.precalcNextBuf(dr.ctx); err != nil { return 0, err @@ -220,12 +221,14 @@ func (dr *pbDagReader) WriteTo(w io.Writer) (int64, error) { } } -func (dr *pbDagReader) Close() error { +// Close closes the reader. +func (dr *PBDagReader) Close() error { dr.cancel() return nil } -func (dr *pbDagReader) Offset() int64 { +// Offset returns the current reader offset +func (dr *PBDagReader) Offset() int64 { return dr.offset } @@ -233,7 +236,7 @@ func (dr *pbDagReader) Offset() int64 { // interface matches standard unix seek // TODO: check if we can do relative seeks, to reduce the amount of dagreader // recreations that need to happen. -func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) { +func (dr *PBDagReader) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: if offset < 0 { @@ -253,17 +256,17 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) { if dr.buf != nil { dr.buf.Close() } - dr.buf = NewBufDagReader(pb.GetData()[offset:]) + dr.buf = newBufDagReader(pb.GetData()[offset:]) // start reading links from the beginning dr.linkPosition = 0 dr.offset = offset return offset, nil - } else { - // skip past root block data - left -= int64(len(pb.Data)) } + // skip past root block data + left -= int64(len(pb.Data)) + // iterate through links and find where we need to be for i := 0; i < len(pb.Blocksizes); i++ { if pb.Blocksizes[i] > uint64(left) { From 8bb45fda8ea250fce6eb351ece8a13d749742adc Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 19:14:48 +0100 Subject: [PATCH 06/10] Golint: make BufDagReader public License: MIT Signed-off-by: Hector Sanjuan --- unixfs/io/bufdagreader.go | 24 +++++++++++++++--------- unixfs/io/dagreader.go | 2 +- unixfs/io/pbdagreader.go | 6 +++--- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/unixfs/io/bufdagreader.go b/unixfs/io/bufdagreader.go index 8c6fd0cf8..6074099e8 100644 --- a/unixfs/io/bufdagreader.go +++ b/unixfs/io/bufdagreader.go @@ -6,27 +6,32 @@ import ( "io" ) -type bufDagReader struct { +// BufDagReader implements a DagReader that reads from a byte slice +// using a bytes.Reader. It is used for RawNodes. +type BufDagReader struct { *bytes.Reader } -// newBufDagReader returns a DAG reader for the given byte slice. +// NewBufDagReader returns a DAG reader for the given byte slice. // BufDagReader is used to read RawNodes. -func newBufDagReader(b []byte) *bufDagReader { - return &bufDagReader{bytes.NewReader(b)} +func NewBufDagReader(b []byte) *BufDagReader { + return &BufDagReader{bytes.NewReader(b)} } -var _ DagReader = (*bufDagReader)(nil) +var _ DagReader = (*BufDagReader)(nil) -func (*bufDagReader) Close() error { +// Close is a nop. +func (*BufDagReader) Close() error { return nil } -func (rd *bufDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) { +// CtxReadFull reads the slice onto b. +func (rd *BufDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) { return rd.Read(b) } -func (rd *bufDagReader) Offset() int64 { +// Offset returns the current offset. +func (rd *BufDagReader) Offset() int64 { of, err := rd.Seek(0, io.SeekCurrent) if err != nil { panic("this should never happen " + err.Error()) @@ -34,7 +39,8 @@ func (rd *bufDagReader) Offset() int64 { return of } -func (rd *bufDagReader) Size() uint64 { +// Size returns the size of the buffer. +func (rd *BufDagReader) Size() uint64 { s := rd.Reader.Size() if s < 0 { panic("size smaller than 0 (impossible!!)") diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index 037e0a256..ec3b3e028 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -43,7 +43,7 @@ type ReadSeekCloser interface { func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { switch n := n.(type) { case *mdag.RawNode: - return newBufDagReader(n.RawData()), nil + return NewBufDagReader(n.RawData()), nil case *mdag.ProtoNode: pb := new(ftpb.Data) if err := proto.Unmarshal(n.Data(), pb); err != nil { diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 4523481b3..ce3abf439 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -57,7 +57,7 @@ func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv return &PBDagReader{ node: n, serv: serv, - buf: newBufDagReader(pb.GetData()), + buf: NewBufDagReader(pb.GetData()), promises: make([]*ipld.NodePromise, len(curLinks)), links: curLinks, ctx: fctx, @@ -119,7 +119,7 @@ func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error { dr.buf = NewPBFileReader(dr.ctx, nxt, pb, dr.serv) return nil case ftpb.Data_Raw: - dr.buf = newBufDagReader(pb.GetData()) + dr.buf = NewBufDagReader(pb.GetData()) return nil case ftpb.Data_Metadata: return errors.New("shouldnt have had metadata object inside file") @@ -256,7 +256,7 @@ func (dr *PBDagReader) Seek(offset int64, whence int) (int64, error) { if dr.buf != nil { dr.buf.Close() } - dr.buf = newBufDagReader(pb.GetData()[offset:]) + dr.buf = NewBufDagReader(pb.GetData()[offset:]) // start reading links from the beginning dr.linkPosition = 0 From 79baf42e1e315b3ca064cda6991fa99ac24986e1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 19:17:59 +0100 Subject: [PATCH 07/10] Golint: improve io.Find documentation per @stebalien's comment License: MIT Signed-off-by: Hector Sanjuan --- unixfs/io/dirbuilder.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index f2dee053c..616617dee 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -162,9 +162,8 @@ func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) { return d.shard.EnumLinks(ctx) } -// Find returns the ipld.Node with the given name, if it is contained in this -// directory. Find only searches in the most inmediate links, and not -// recursively in the tree. +// Find returns the root node of the file named 'name' within this directory. +// In the case of HAMT-directories, it will traverse the tree. func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) { if d.shard == nil { lnk, err := d.dirnode.GetNodeLink(name) From 576872076c0510f6e6d14851f8d27b81d5e571d9 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 6 Feb 2018 19:19:30 +0100 Subject: [PATCH 08/10] unixfs/mod: use errors.New() for all defined errors License: MIT Signed-off-by: Hector Sanjuan --- unixfs/mod/dagmodifier.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index 05c5c3587..dfadd778b 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "errors" - "fmt" "io" chunk "github.com/ipfs/go-ipfs/importer/chunk" @@ -25,7 +24,7 @@ import ( var ( ErrSeekFail = errors.New("failed to seek properly") ErrUnrecognizedWhence = errors.New("unrecognized whence") - ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)") + ErrNotUnixfs = errors.New("dagmodifier only supports unixfs nodes (proto or raw)") ) // 2MB From 3ec869fee77bd52a84746cd9de487c625f92a532 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 7 Feb 2018 10:06:59 +0100 Subject: [PATCH 09/10] Golint: improve unixfs/dagreader.go comments Per @whys suggestions License: MIT Signed-off-by: Hector Sanjuan --- unixfs/io/dagreader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index ec3b3e028..53e990297 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -20,8 +20,8 @@ var ( ErrCantReadSymlinks = errors.New("cannot currently read symlinks") ) -// A DagReader represents a ReadSeekCloser which offers additional methods -// like Size. Different implementations of readers are used for the different +// // A DagReader provides read-only read and seek acess to a unixfs file. +// Different implementations of readers are used for the different // types of unixfs/protobuf-encoded nodes. type DagReader interface { ReadSeekCloser @@ -30,7 +30,7 @@ type DagReader interface { Offset() int64 } -// A ReadSeekCloser implements interfaces to read, write, seek and close. +// A ReadSeekCloser implements interfaces to read, copy, seek and close. type ReadSeekCloser interface { io.Reader io.Seeker From f4f450d774f5394f543a134d3887b5c9e27c4b42 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 7 Feb 2018 12:33:58 +0100 Subject: [PATCH 10/10] Golint: remove extra // in comment License: MIT Signed-off-by: Hector Sanjuan --- unixfs/io/dagreader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index 53e990297..af3fbb330 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -20,7 +20,7 @@ var ( ErrCantReadSymlinks = errors.New("cannot currently read symlinks") ) -// // A DagReader provides read-only read and seek acess to a unixfs file. +// A DagReader provides read-only read and seek acess to a unixfs file. // Different implementations of readers are used for the different // types of unixfs/protobuf-encoded nodes. type DagReader interface {