diff --git a/core/coreiface/block.go b/core/coreiface/block.go new file mode 100644 index 000000000..dbe31e9f8 --- /dev/null +++ b/core/coreiface/block.go @@ -0,0 +1,38 @@ +package iface + +import ( + "context" + "io" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" +) + +// BlockStat contains information about a block +type BlockStat interface { + // Size is the size of a block + Size() int + + // Path returns path to the block + Path() path.Resolved +} + +// BlockAPI specifies the interface to the block layer +type BlockAPI interface { + // Put imports raw block data, hashing it using specified settings. + Put(context.Context, io.Reader, ...options.BlockPutOption) (BlockStat, error) + + // Get attempts to resolve the path and return a reader for data in the block + Get(context.Context, path.Path) (io.Reader, error) + + // Rm removes the block specified by the path from local blockstore. + // By default an error will be returned if the block can't be found locally. + // + // NOTE: If the specified block is pinned it won't be removed and no error + // will be returned + Rm(context.Context, path.Path, ...options.BlockRmOption) error + + // Stat returns information on + Stat(context.Context, path.Path) (BlockStat, error) +} diff --git a/core/coreiface/coreapi.go b/core/coreiface/coreapi.go new file mode 100644 index 000000000..7276a3f60 --- /dev/null +++ b/core/coreiface/coreapi.go @@ -0,0 +1,60 @@ +// Package iface defines IPFS Core API which is a set of interfaces used to +// interact with IPFS nodes. +package iface + +import ( + "context" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" + + ipld "github.com/ipfs/go-ipld-format" +) + +// CoreAPI defines an unified interface to IPFS for Go programs +type CoreAPI interface { + // Unixfs returns an implementation of Unixfs API + Unixfs() UnixfsAPI + + // Block returns an implementation of Block API + Block() BlockAPI + + // Dag returns an implementation of Dag API + Dag() APIDagService + + // Name returns an implementation of Name API + Name() NameAPI + + // Key returns an implementation of Key API + Key() KeyAPI + + // Pin returns an implementation of Pin API + Pin() PinAPI + + // Object returns an implementation of Object API + Object() ObjectAPI + + // Dht returns an implementation of Dht API + Dht() DhtAPI + + // Swarm returns an implementation of Swarm API + Swarm() SwarmAPI + + // PubSub returns an implementation of PubSub API + PubSub() PubSubAPI + + // Routing returns an implementation of Routing API + Routing() RoutingAPI + + // ResolvePath resolves the path using Unixfs resolver + ResolvePath(context.Context, path.Path) (path.Resolved, error) + + // ResolveNode resolves the path (if not resolved already) using Unixfs + // resolver, gets and returns the resolved Node + ResolveNode(context.Context, path.Path) (ipld.Node, error) + + // WithOptions creates new instance of CoreAPI based on this instance with + // a set of options applied + WithOptions(...options.ApiOption) (CoreAPI, error) +} diff --git a/core/coreiface/dag.go b/core/coreiface/dag.go new file mode 100644 index 000000000..3cc3aeb4d --- /dev/null +++ b/core/coreiface/dag.go @@ -0,0 +1,13 @@ +package iface + +import ( + ipld "github.com/ipfs/go-ipld-format" +) + +// APIDagService extends ipld.DAGService +type APIDagService interface { + ipld.DAGService + + // Pinning returns special NodeAdder which recursively pins added nodes + Pinning() ipld.NodeAdder +} diff --git a/core/coreiface/dht.go b/core/coreiface/dht.go new file mode 100644 index 000000000..93027a406 --- /dev/null +++ b/core/coreiface/dht.go @@ -0,0 +1,27 @@ +package iface + +import ( + "context" + + "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// DhtAPI specifies the interface to the DHT +// Note: This API will likely get deprecated in near future, see +// https://github.com/ipfs/interface-ipfs-core/issues/249 for more context. +type DhtAPI interface { + // FindPeer queries the DHT for all of the multiaddresses associated with a + // Peer ID + FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) + + // FindProviders finds peers in the DHT who can provide a specific value + // given a key. + FindProviders(context.Context, path.Path, ...options.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) + + // Provide announces to the network that you are providing given values + Provide(context.Context, path.Path, ...options.DhtProvideOption) error +} diff --git a/core/coreiface/errors.go b/core/coreiface/errors.go new file mode 100644 index 000000000..e0bd7805d --- /dev/null +++ b/core/coreiface/errors.go @@ -0,0 +1,10 @@ +package iface + +import "errors" + +var ( + ErrIsDir = errors.New("this dag node is a directory") + ErrNotFile = errors.New("this dag node is not a regular file") + ErrOffline = errors.New("this action must be run in online mode, try running 'ipfs daemon' first") + ErrNotSupported = errors.New("operation not supported") +) diff --git a/core/coreiface/idfmt.go b/core/coreiface/idfmt.go new file mode 100644 index 000000000..80fd0f822 --- /dev/null +++ b/core/coreiface/idfmt.go @@ -0,0 +1,19 @@ +package iface + +import ( + "github.com/libp2p/go-libp2p/core/peer" + mbase "github.com/multiformats/go-multibase" +) + +func FormatKeyID(id peer.ID) string { + if s, err := peer.ToCid(id).StringOfBase(mbase.Base36); err != nil { + panic(err) + } else { + return s + } +} + +// FormatKey formats the given IPNS key in a canonical way. +func FormatKey(key Key) string { + return FormatKeyID(key.ID()) +} diff --git a/core/coreiface/key.go b/core/coreiface/key.go new file mode 100644 index 000000000..118fe2e4f --- /dev/null +++ b/core/coreiface/key.go @@ -0,0 +1,43 @@ +package iface + +import ( + "context" + + "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// Key specifies the interface to Keys in KeyAPI Keystore +type Key interface { + // Key returns key name + Name() string + + // Path returns key path + Path() path.Path + + // ID returns key PeerID + ID() peer.ID +} + +// KeyAPI specifies the interface to Keystore +type KeyAPI interface { + // Generate generates new key, stores it in the keystore under the specified + // name and returns a base58 encoded multihash of it's public key + Generate(ctx context.Context, name string, opts ...options.KeyGenerateOption) (Key, error) + + // Rename renames oldName key to newName. Returns the key and whether another + // key was overwritten, or an error + Rename(ctx context.Context, oldName string, newName string, opts ...options.KeyRenameOption) (Key, bool, error) + + // List lists keys stored in keystore + List(ctx context.Context) ([]Key, error) + + // Self returns the 'main' node key + Self(ctx context.Context) (Key, error) + + // Remove removes keys from keystore. Returns ipns path of the removed key + Remove(ctx context.Context, name string) (Key, error) +} diff --git a/core/coreiface/name.go b/core/coreiface/name.go new file mode 100644 index 000000000..0c06183e6 --- /dev/null +++ b/core/coreiface/name.go @@ -0,0 +1,48 @@ +package iface + +import ( + "context" + "errors" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" +) + +var ErrResolveFailed = errors.New("could not resolve name") + +// IpnsEntry specifies the interface to IpnsEntries +type IpnsEntry interface { + // Name returns IpnsEntry name + Name() string + // Value returns IpnsEntry value + Value() path.Path +} + +type IpnsResult struct { + path.Path + Err error +} + +// NameAPI specifies the interface to IPNS. +// +// IPNS is a PKI namespace, where names are the hashes of public keys, and the +// private key enables publishing new (signed) values. In both publish and +// resolve, the default name used is the node's own PeerID, which is the hash of +// its public key. +// +// You can use .Key API to list and generate more names and their respective keys. +type NameAPI interface { + // Publish announces new IPNS name + Publish(ctx context.Context, path path.Path, opts ...options.NamePublishOption) (IpnsEntry, error) + + // Resolve attempts to resolve the newest version of the specified name + Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (path.Path, error) + + // Search is a version of Resolve which outputs paths as they are discovered, + // reducing the time to first entry + // + // Note: by default, all paths read from the channel are considered unsafe, + // except the latest (last path in channel read buffer). + Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan IpnsResult, error) +} diff --git a/core/coreiface/object.go b/core/coreiface/object.go new file mode 100644 index 000000000..d983fa49b --- /dev/null +++ b/core/coreiface/object.go @@ -0,0 +1,108 @@ +package iface + +import ( + "context" + "io" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +// ObjectStat provides information about dag nodes +type ObjectStat struct { + // Cid is the CID of the node + Cid cid.Cid + + // NumLinks is number of links the node contains + NumLinks int + + // BlockSize is size of the raw serialized node + BlockSize int + + // LinksSize is size of the links block section + LinksSize int + + // DataSize is the size of data block section + DataSize int + + // CumulativeSize is size of the tree (BlockSize + link sizes) + CumulativeSize int +} + +// ChangeType denotes type of change in ObjectChange +type ChangeType int + +const ( + // DiffAdd is set when a link was added to the graph + DiffAdd ChangeType = iota + + // DiffRemove is set when a link was removed from the graph + DiffRemove + + // DiffMod is set when a link was changed in the graph + DiffMod +) + +// ObjectChange represents a change ia a graph +type ObjectChange struct { + // Type of the change, either: + // * DiffAdd - Added a link + // * DiffRemove - Removed a link + // * DiffMod - Modified a link + Type ChangeType + + // Path to the changed link + Path string + + // Before holds the link path before the change. Note that when a link is + // added, this will be nil. + Before path.Resolved + + // After holds the link path after the change. Note that when a link is + // removed, this will be nil. + After path.Resolved +} + +// ObjectAPI specifies the interface to MerkleDAG and contains useful utilities +// for manipulating MerkleDAG data structures. +type ObjectAPI interface { + // New creates new, empty (by default) dag-node. + New(context.Context, ...options.ObjectNewOption) (ipld.Node, error) + + // Put imports the data into merkledag + Put(context.Context, io.Reader, ...options.ObjectPutOption) (path.Resolved, error) + + // Get returns the node for the path + Get(context.Context, path.Path) (ipld.Node, error) + + // Data returns reader for data of the node + Data(context.Context, path.Path) (io.Reader, error) + + // Links returns lint or links the node contains + Links(context.Context, path.Path) ([]*ipld.Link, error) + + // Stat returns information about the node + Stat(context.Context, path.Path) (*ObjectStat, error) + + // AddLink adds a link under the specified path. child path can point to a + // subdirectory within the patent which must be present (can be overridden + // with WithCreate option). + AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...options.ObjectAddLinkOption) (path.Resolved, error) + + // RmLink removes a link from the node + RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) + + // AppendData appends data to the node + AppendData(context.Context, path.Path, io.Reader) (path.Resolved, error) + + // SetData sets the data contained in the node + SetData(context.Context, path.Path, io.Reader) (path.Resolved, error) + + // Diff returns a set of changes needed to transform the first object into the + // second. + Diff(context.Context, path.Path, path.Path) ([]ObjectChange, error) +} diff --git a/core/coreiface/options/block.go b/core/coreiface/options/block.go new file mode 100644 index 000000000..130648682 --- /dev/null +++ b/core/coreiface/options/block.go @@ -0,0 +1,164 @@ +package options + +import ( + "fmt" + + cid "github.com/ipfs/go-cid" + mc "github.com/multiformats/go-multicodec" + mh "github.com/multiformats/go-multihash" +) + +type BlockPutSettings struct { + CidPrefix cid.Prefix + Pin bool +} + +type BlockRmSettings struct { + Force bool +} + +type BlockPutOption func(*BlockPutSettings) error +type BlockRmOption func(*BlockRmSettings) error + +func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) { + var cidPrefix cid.Prefix + + // Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts) + cidPrefix.Version = 1 + cidPrefix.Codec = uint64(mc.Raw) + cidPrefix.MhType = mh.SHA2_256 + cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum() + + options := &BlockPutSettings{ + CidPrefix: cidPrefix, + Pin: false, + } + + // Apply any overrides + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) { + options := &BlockRmSettings{ + Force: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type blockOpts struct{} + +var Block blockOpts + +// CidCodec is the modern option for Block.Put which specifies the multicodec to use +// in the CID returned by the Block.Put operation. +// It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default. +func (blockOpts) CidCodec(codecName string) BlockPutOption { + return func(settings *BlockPutSettings) error { + if codecName == "" { + return nil + } + code, err := codeFromName(codecName) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + return nil + } +} + +// Map string to code from go-multicodec +func codeFromName(codecName string) (mc.Code, error) { + var cidCodec mc.Code + err := cidCodec.Set(codecName) + return cidCodec, err +} + +// Format is a legacy option for Block.Put which specifies the multicodec to +// use to serialize the object. +// Provided for backward-compatibility only. Use CidCodec instead. +func (blockOpts) Format(format string) BlockPutOption { + return func(settings *BlockPutSettings) error { + if format == "" { + return nil + } + // Opt-in CIDv0 support for backward-compatibility + if format == "v0" { + settings.CidPrefix.Version = 0 + } + + // Fixup a legacy (invalid) names for dag-pb (0x70) + if format == "v0" || format == "protobuf" { + format = "dag-pb" + } + + // Fixup invalid name for dag-cbor (0x71) + if format == "cbor" { + format = "dag-cbor" + } + + // Set code based on name passed as "format" + code, err := codeFromName(format) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + + // If CIDv0, ensure all parameters are compatible + // (in theory go-cid would validate this anyway, but we want to provide better errors) + pref := settings.CidPrefix + if pref.Version == 0 { + if pref.Codec != uint64(mc.DagPb) { + return fmt.Errorf("only dag-pb is allowed with CIDv0") + } + if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) { + return fmt.Errorf("only sha2-255-32 is allowed with CIDv0") + } + } + + return nil + } + +} + +// Hash is an option for Block.Put which specifies the multihash settings to use +// when hashing the object. Default is mh.SHA2_256 (0x12). +// If mhLen is set to -1, default length for the hash will be used +func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.CidPrefix.MhType = mhType + settings.CidPrefix.MhLength = mhLen + return nil + } +} + +// Pin is an option for Block.Put which specifies whether to (recursively) pin +// added blocks +func (blockOpts) Pin(pin bool) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.Pin = pin + return nil + } +} + +// Force is an option for Block.Rm which, when set to true, will ignore +// non-existing blocks +func (blockOpts) Force(force bool) BlockRmOption { + return func(settings *BlockRmSettings) error { + settings.Force = force + return nil + } +} diff --git a/core/coreiface/options/dht.go b/core/coreiface/options/dht.go new file mode 100644 index 000000000..e13e16020 --- /dev/null +++ b/core/coreiface/options/dht.go @@ -0,0 +1,62 @@ +package options + +type DhtProvideSettings struct { + Recursive bool +} + +type DhtFindProvidersSettings struct { + NumProviders int +} + +type DhtProvideOption func(*DhtProvideSettings) error +type DhtFindProvidersOption func(*DhtFindProvidersSettings) error + +func DhtProvideOptions(opts ...DhtProvideOption) (*DhtProvideSettings, error) { + options := &DhtProvideSettings{ + Recursive: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func DhtFindProvidersOptions(opts ...DhtFindProvidersOption) (*DhtFindProvidersSettings, error) { + options := &DhtFindProvidersSettings{ + NumProviders: 20, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type dhtOpts struct{} + +var Dht dhtOpts + +// Recursive is an option for Dht.Provide which specifies whether to provide +// the given path recursively +func (dhtOpts) Recursive(recursive bool) DhtProvideOption { + return func(settings *DhtProvideSettings) error { + settings.Recursive = recursive + return nil + } +} + +// NumProviders is an option for Dht.FindProviders which specifies the +// number of peers to look for. Default is 20 +func (dhtOpts) NumProviders(numProviders int) DhtFindProvidersOption { + return func(settings *DhtFindProvidersSettings) error { + settings.NumProviders = numProviders + return nil + } +} diff --git a/core/coreiface/options/global.go b/core/coreiface/options/global.go new file mode 100644 index 000000000..90e2586f1 --- /dev/null +++ b/core/coreiface/options/global.go @@ -0,0 +1,47 @@ +package options + +type ApiSettings struct { + Offline bool + FetchBlocks bool +} + +type ApiOption func(*ApiSettings) error + +func ApiOptions(opts ...ApiOption) (*ApiSettings, error) { + options := &ApiSettings{ + Offline: false, + FetchBlocks: true, + } + + return ApiOptionsTo(options, opts...) +} + +func ApiOptionsTo(options *ApiSettings, opts ...ApiOption) (*ApiSettings, error) { + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type apiOpts struct{} + +var Api apiOpts + +func (apiOpts) Offline(offline bool) ApiOption { + return func(settings *ApiSettings) error { + settings.Offline = offline + return nil + } +} + +// FetchBlocks when set to false prevents api from fetching blocks from the +// network while allowing other services such as IPNS to still be online +func (apiOpts) FetchBlocks(fetch bool) ApiOption { + return func(settings *ApiSettings) error { + settings.FetchBlocks = fetch + return nil + } +} diff --git a/core/coreiface/options/key.go b/core/coreiface/options/key.go new file mode 100644 index 000000000..4bc53a65f --- /dev/null +++ b/core/coreiface/options/key.go @@ -0,0 +1,87 @@ +package options + +const ( + RSAKey = "rsa" + Ed25519Key = "ed25519" + + DefaultRSALen = 2048 +) + +type KeyGenerateSettings struct { + Algorithm string + Size int +} + +type KeyRenameSettings struct { + Force bool +} + +type KeyGenerateOption func(*KeyGenerateSettings) error +type KeyRenameOption func(*KeyRenameSettings) error + +func KeyGenerateOptions(opts ...KeyGenerateOption) (*KeyGenerateSettings, error) { + options := &KeyGenerateSettings{ + Algorithm: RSAKey, + Size: -1, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func KeyRenameOptions(opts ...KeyRenameOption) (*KeyRenameSettings, error) { + options := &KeyRenameSettings{ + Force: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type keyOpts struct{} + +var Key keyOpts + +// Type is an option for Key.Generate which specifies which algorithm +// should be used for the key. Default is options.RSAKey +// +// Supported key types: +// * options.RSAKey +// * options.Ed25519Key +func (keyOpts) Type(algorithm string) KeyGenerateOption { + return func(settings *KeyGenerateSettings) error { + settings.Algorithm = algorithm + return nil + } +} + +// Size is an option for Key.Generate which specifies the size of the key to +// generated. Default is -1 +// +// value of -1 means 'use default size for key type': +// - 2048 for RSA +func (keyOpts) Size(size int) KeyGenerateOption { + return func(settings *KeyGenerateSettings) error { + settings.Size = size + return nil + } +} + +// Force is an option for Key.Rename which specifies whether to allow to +// replace existing keys. +func (keyOpts) Force(force bool) KeyRenameOption { + return func(settings *KeyRenameSettings) error { + settings.Force = force + return nil + } +} diff --git a/core/coreiface/options/name.go b/core/coreiface/options/name.go new file mode 100644 index 000000000..ae8be9ae9 --- /dev/null +++ b/core/coreiface/options/name.go @@ -0,0 +1,121 @@ +package options + +import ( + "time" + + ropts "github.com/ipfs/boxo/coreiface/options/namesys" +) + +const ( + DefaultNameValidTime = 24 * time.Hour +) + +type NamePublishSettings struct { + ValidTime time.Duration + Key string + + TTL *time.Duration + + AllowOffline bool +} + +type NameResolveSettings struct { + Cache bool + + ResolveOpts []ropts.ResolveOpt +} + +type NamePublishOption func(*NamePublishSettings) error +type NameResolveOption func(*NameResolveSettings) error + +func NamePublishOptions(opts ...NamePublishOption) (*NamePublishSettings, error) { + options := &NamePublishSettings{ + ValidTime: DefaultNameValidTime, + Key: "self", + + AllowOffline: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +func NameResolveOptions(opts ...NameResolveOption) (*NameResolveSettings, error) { + options := &NameResolveSettings{ + Cache: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type nameOpts struct{} + +var Name nameOpts + +// ValidTime is an option for Name.Publish which specifies for how long the +// entry will remain valid. Default value is 24h +func (nameOpts) ValidTime(validTime time.Duration) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.ValidTime = validTime + return nil + } +} + +// Key is an option for Name.Publish which specifies the key to use for +// publishing. Default value is "self" which is the node's own PeerID. +// The key parameter must be either PeerID or keystore key alias. +// +// You can use KeyAPI to list and generate more names and their respective keys. +func (nameOpts) Key(key string) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.Key = key + return nil + } +} + +// AllowOffline is an option for Name.Publish which specifies whether to allow +// publishing when the node is offline. Default value is false +func (nameOpts) AllowOffline(allow bool) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.AllowOffline = allow + return nil + } +} + +// TTL is an option for Name.Publish which specifies the time duration the +// published record should be cached for (caution: experimental). +func (nameOpts) TTL(ttl time.Duration) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.TTL = &ttl + return nil + } +} + +// Cache is an option for Name.Resolve which specifies if cache should be used. +// Default value is true +func (nameOpts) Cache(cache bool) NameResolveOption { + return func(settings *NameResolveSettings) error { + settings.Cache = cache + return nil + } +} + +func (nameOpts) ResolveOption(opt ropts.ResolveOpt) NameResolveOption { + return func(settings *NameResolveSettings) error { + settings.ResolveOpts = append(settings.ResolveOpts, opt) + return nil + } +} diff --git a/core/coreiface/options/namesys/opts.go b/core/coreiface/options/namesys/opts.go new file mode 100644 index 000000000..0cd1ba778 --- /dev/null +++ b/core/coreiface/options/namesys/opts.go @@ -0,0 +1,123 @@ +package nsopts + +import ( + "time" +) + +const ( + // DefaultDepthLimit is the default depth limit used by Resolve. + DefaultDepthLimit = 32 + + // UnlimitedDepth allows infinite recursion in Resolve. You + // probably don't want to use this, but it's here if you absolutely + // trust resolution to eventually complete and can't put an upper + // limit on how many steps it will take. + UnlimitedDepth = 0 + + // DefaultIPNSRecordTTL specifies the time that the record can be cached + // before checking if its validity again. + DefaultIPNSRecordTTL = time.Minute + + // DefaultIPNSRecordEOL specifies the time that the network will cache IPNS + // records after being published. Records should be re-published before this + // interval expires. We use the same default expiration as the DHT. + DefaultIPNSRecordEOL = 48 * time.Hour +) + +// ResolveOpts specifies options for resolving an IPNS path +type ResolveOpts struct { + // Recursion depth limit + Depth uint + // The number of IPNS records to retrieve from the DHT + // (the best record is selected from this set) + DhtRecordCount uint + // The amount of time to wait for DHT records to be fetched + // and verified. A zero value indicates that there is no explicit + // timeout (although there is an implicit timeout due to dial + // timeouts within the DHT) + DhtTimeout time.Duration +} + +// DefaultResolveOpts returns the default options for resolving +// an IPNS path +func DefaultResolveOpts() ResolveOpts { + return ResolveOpts{ + Depth: DefaultDepthLimit, + DhtRecordCount: 16, + DhtTimeout: time.Minute, + } +} + +// ResolveOpt is used to set an option +type ResolveOpt func(*ResolveOpts) + +// Depth is the recursion depth limit +func Depth(depth uint) ResolveOpt { + return func(o *ResolveOpts) { + o.Depth = depth + } +} + +// DhtRecordCount is the number of IPNS records to retrieve from the DHT +func DhtRecordCount(count uint) ResolveOpt { + return func(o *ResolveOpts) { + o.DhtRecordCount = count + } +} + +// DhtTimeout is the amount of time to wait for DHT records to be fetched +// and verified. A zero value indicates that there is no explicit timeout +func DhtTimeout(timeout time.Duration) ResolveOpt { + return func(o *ResolveOpts) { + o.DhtTimeout = timeout + } +} + +// ProcessOpts converts an array of ResolveOpt into a ResolveOpts object +func ProcessOpts(opts []ResolveOpt) ResolveOpts { + rsopts := DefaultResolveOpts() + for _, option := range opts { + option(&rsopts) + } + return rsopts +} + +// PublishOptions specifies options for publishing an IPNS record. +type PublishOptions struct { + EOL time.Time + TTL time.Duration +} + +// DefaultPublishOptions returns the default options for publishing an IPNS record. +func DefaultPublishOptions() PublishOptions { + return PublishOptions{ + EOL: time.Now().Add(DefaultIPNSRecordEOL), + TTL: DefaultIPNSRecordTTL, + } +} + +// PublishOption is used to set an option for PublishOpts. +type PublishOption func(*PublishOptions) + +// PublishWithEOL sets an EOL. +func PublishWithEOL(eol time.Time) PublishOption { + return func(o *PublishOptions) { + o.EOL = eol + } +} + +// PublishWithEOL sets a TTL. +func PublishWithTTL(ttl time.Duration) PublishOption { + return func(o *PublishOptions) { + o.TTL = ttl + } +} + +// ProcessPublishOptions converts an array of PublishOpt into a PublishOpts object. +func ProcessPublishOptions(opts []PublishOption) PublishOptions { + rsopts := DefaultPublishOptions() + for _, option := range opts { + option(&rsopts) + } + return rsopts +} diff --git a/core/coreiface/options/object.go b/core/coreiface/options/object.go new file mode 100644 index 000000000..e484a9f36 --- /dev/null +++ b/core/coreiface/options/object.go @@ -0,0 +1,124 @@ +package options + +type ObjectNewSettings struct { + Type string +} + +type ObjectPutSettings struct { + InputEnc string + DataType string + Pin bool +} + +type ObjectAddLinkSettings struct { + Create bool +} + +type ObjectNewOption func(*ObjectNewSettings) error +type ObjectPutOption func(*ObjectPutSettings) error +type ObjectAddLinkOption func(*ObjectAddLinkSettings) error + +func ObjectNewOptions(opts ...ObjectNewOption) (*ObjectNewSettings, error) { + options := &ObjectNewSettings{ + Type: "empty", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func ObjectPutOptions(opts ...ObjectPutOption) (*ObjectPutSettings, error) { + options := &ObjectPutSettings{ + InputEnc: "json", + DataType: "text", + Pin: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func ObjectAddLinkOptions(opts ...ObjectAddLinkOption) (*ObjectAddLinkSettings, error) { + options := &ObjectAddLinkSettings{ + Create: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type objectOpts struct{} + +var Object objectOpts + +// Type is an option for Object.New which allows to change the type of created +// dag node. +// +// Supported types: +// * 'empty' - Empty node +// * 'unixfs-dir' - Empty UnixFS directory +func (objectOpts) Type(t string) ObjectNewOption { + return func(settings *ObjectNewSettings) error { + settings.Type = t + return nil + } +} + +// InputEnc is an option for Object.Put which specifies the input encoding of the +// data. Default is "json". +// +// Supported encodings: +// * "protobuf" +// * "json" +func (objectOpts) InputEnc(e string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.InputEnc = e + return nil + } +} + +// DataType is an option for Object.Put which specifies the encoding of data +// field when using Json or XML input encoding. +// +// Supported types: +// * "text" (default) +// * "base64" +func (objectOpts) DataType(t string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.DataType = t + return nil + } +} + +// Pin is an option for Object.Put which specifies whether to pin the added +// objects, default is false +func (objectOpts) Pin(pin bool) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.Pin = pin + return nil + } +} + +// Create is an option for Object.AddLink which specifies whether create required +// directories for the child +func (objectOpts) Create(create bool) ObjectAddLinkOption { + return func(settings *ObjectAddLinkSettings) error { + settings.Create = create + return nil + } +} diff --git a/core/coreiface/options/pin.go b/core/coreiface/options/pin.go new file mode 100644 index 000000000..75c2b8a26 --- /dev/null +++ b/core/coreiface/options/pin.go @@ -0,0 +1,283 @@ +package options + +import "fmt" + +// PinAddSettings represent the settings for PinAPI.Add +type PinAddSettings struct { + Recursive bool +} + +// PinLsSettings represent the settings for PinAPI.Ls +type PinLsSettings struct { + Type string +} + +// PinIsPinnedSettings represent the settings for PinAPI.IsPinned +type PinIsPinnedSettings struct { + WithType string +} + +// PinRmSettings represents the settings for PinAPI.Rm +type PinRmSettings struct { + Recursive bool +} + +// PinUpdateSettings represent the settings for PinAPI.Update +type PinUpdateSettings struct { + Unpin bool +} + +// PinAddOption is the signature of an option for PinAPI.Add +type PinAddOption func(*PinAddSettings) error + +// PinLsOption is the signature of an option for PinAPI.Ls +type PinLsOption func(*PinLsSettings) error + +// PinIsPinnedOption is the signature of an option for PinAPI.IsPinned +type PinIsPinnedOption func(*PinIsPinnedSettings) error + +// PinRmOption is the signature of an option for PinAPI.Rm +type PinRmOption func(*PinRmSettings) error + +// PinUpdateOption is the signature of an option for PinAPI.Update +type PinUpdateOption func(*PinUpdateSettings) error + +// PinAddOptions compile a series of PinAddOption into a ready to use +// PinAddSettings and set the default values. +func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { + options := &PinAddSettings{ + Recursive: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinLsOptions compile a series of PinLsOption into a ready to use +// PinLsSettings and set the default values. +func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { + options := &PinLsSettings{ + Type: "all", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinIsPinnedOptions compile a series of PinIsPinnedOption into a ready to use +// PinIsPinnedSettings and set the default values. +func PinIsPinnedOptions(opts ...PinIsPinnedOption) (*PinIsPinnedSettings, error) { + options := &PinIsPinnedSettings{ + WithType: "all", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinRmOptions compile a series of PinRmOption into a ready to use +// PinRmSettings and set the default values. +func PinRmOptions(opts ...PinRmOption) (*PinRmSettings, error) { + options := &PinRmSettings{ + Recursive: true, + } + + for _, opt := range opts { + if err := opt(options); err != nil { + return nil, err + } + } + + return options, nil +} + +// PinUpdateOptions compile a series of PinUpdateOption into a ready to use +// PinUpdateSettings and set the default values. +func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { + options := &PinUpdateSettings{ + Unpin: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type pinOpts struct { + Ls pinLsOpts + IsPinned pinIsPinnedOpts +} + +// Pin provide an access to all the options for the Pin API. +var Pin pinOpts + +type pinLsOpts struct{} + +// All is an option for Pin.Ls which will make it return all pins. It is +// the default +func (pinLsOpts) All() PinLsOption { + return Pin.Ls.pinType("all") +} + +// Recursive is an option for Pin.Ls which will make it only return recursive +// pins +func (pinLsOpts) Recursive() PinLsOption { + return Pin.Ls.pinType("recursive") +} + +// Direct is an option for Pin.Ls which will make it only return direct (non +// recursive) pins +func (pinLsOpts) Direct() PinLsOption { + return Pin.Ls.pinType("direct") +} + +// Indirect is an option for Pin.Ls which will make it only return indirect pins +// (objects referenced by other recursively pinned objects) +func (pinLsOpts) Indirect() PinLsOption { + return Pin.Ls.pinType("indirect") +} + +// Type is an option for Pin.Ls which will make it only return pins of the given +// type. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinLsOpts) Type(typeStr string) (PinLsOption, error) { + switch typeStr { + case "all", "direct", "indirect", "recursive": + return Pin.Ls.pinType(typeStr), nil + default: + return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) + } +} + +// pinType is an option for Pin.Ls which allows to specify which pin types should +// be returned +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinLsOpts) pinType(t string) PinLsOption { + return func(settings *PinLsSettings) error { + settings.Type = t + return nil + } +} + +type pinIsPinnedOpts struct{} + +// All is an option for Pin.IsPinned which will make it search in all type of pins. +// It is the default +func (pinIsPinnedOpts) All() PinIsPinnedOption { + return Pin.IsPinned.pinType("all") +} + +// Recursive is an option for Pin.IsPinned which will make it only search in +// recursive pins +func (pinIsPinnedOpts) Recursive() PinIsPinnedOption { + return Pin.IsPinned.pinType("recursive") +} + +// Direct is an option for Pin.IsPinned which will make it only search in direct +// (non recursive) pins +func (pinIsPinnedOpts) Direct() PinIsPinnedOption { + return Pin.IsPinned.pinType("direct") +} + +// Indirect is an option for Pin.IsPinned which will make it only search indirect +// pins (objects referenced by other recursively pinned objects) +func (pinIsPinnedOpts) Indirect() PinIsPinnedOption { + return Pin.IsPinned.pinType("indirect") +} + +// Type is an option for Pin.IsPinned which will make it only search pins of the given +// type. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinIsPinnedOpts) Type(typeStr string) (PinIsPinnedOption, error) { + switch typeStr { + case "all", "direct", "indirect", "recursive": + return Pin.IsPinned.pinType(typeStr), nil + default: + return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) + } +} + +// pinType is an option for Pin.IsPinned which allows to specify which pin type the given +// pin is expected to be, speeding up the research. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinIsPinnedOpts) pinType(t string) PinIsPinnedOption { + return func(settings *PinIsPinnedSettings) error { + settings.WithType = t + return nil + } +} + +// Recursive is an option for Pin.Add which specifies whether to pin an entire +// object tree or just one object. Default: true +func (pinOpts) Recursive(recursive bool) PinAddOption { + return func(settings *PinAddSettings) error { + settings.Recursive = recursive + return nil + } +} + +// RmRecursive is an option for Pin.Rm which specifies whether to recursively +// unpin the object linked to by the specified object(s). This does not remove +// indirect pins referenced by other recursive pins. +func (pinOpts) RmRecursive(recursive bool) PinRmOption { + return func(settings *PinRmSettings) error { + settings.Recursive = recursive + return nil + } +} + +// Unpin is an option for Pin.Update which specifies whether to remove the old pin. +// Default is true. +func (pinOpts) Unpin(unpin bool) PinUpdateOption { + return func(settings *PinUpdateSettings) error { + settings.Unpin = unpin + return nil + } +} diff --git a/core/coreiface/options/pubsub.go b/core/coreiface/options/pubsub.go new file mode 100644 index 000000000..c387d613d --- /dev/null +++ b/core/coreiface/options/pubsub.go @@ -0,0 +1,58 @@ +package options + +type PubSubPeersSettings struct { + Topic string +} + +type PubSubSubscribeSettings struct { + Discover bool +} + +type PubSubPeersOption func(*PubSubPeersSettings) error +type PubSubSubscribeOption func(*PubSubSubscribeSettings) error + +func PubSubPeersOptions(opts ...PubSubPeersOption) (*PubSubPeersSettings, error) { + options := &PubSubPeersSettings{ + Topic: "", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func PubSubSubscribeOptions(opts ...PubSubSubscribeOption) (*PubSubSubscribeSettings, error) { + options := &PubSubSubscribeSettings{ + Discover: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type pubsubOpts struct{} + +var PubSub pubsubOpts + +func (pubsubOpts) Topic(topic string) PubSubPeersOption { + return func(settings *PubSubPeersSettings) error { + settings.Topic = topic + return nil + } +} + +func (pubsubOpts) Discover(discover bool) PubSubSubscribeOption { + return func(settings *PubSubSubscribeSettings) error { + settings.Discover = discover + return nil + } +} diff --git a/core/coreiface/options/unixfs.go b/core/coreiface/options/unixfs.go new file mode 100644 index 000000000..dd12502e6 --- /dev/null +++ b/core/coreiface/options/unixfs.go @@ -0,0 +1,293 @@ +package options + +import ( + "errors" + "fmt" + + dag "github.com/ipfs/boxo/ipld/merkledag" + cid "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +type Layout int + +const ( + BalancedLayout Layout = iota + TrickleLayout +) + +type UnixfsAddSettings struct { + CidVersion int + MhType uint64 + + Inline bool + InlineLimit int + RawLeaves bool + RawLeavesSet bool + + Chunker string + Layout Layout + + Pin bool + OnlyHash bool + FsCache bool + NoCopy bool + + Events chan<- interface{} + Silent bool + Progress bool +} + +type UnixfsLsSettings struct { + ResolveChildren bool + UseCumulativeSize bool +} + +type UnixfsAddOption func(*UnixfsAddSettings) error +type UnixfsLsOption func(*UnixfsLsSettings) error + +func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) { + options := &UnixfsAddSettings{ + CidVersion: -1, + MhType: mh.SHA2_256, + + Inline: false, + InlineLimit: 32, + RawLeaves: false, + RawLeavesSet: false, + + Chunker: "size-262144", + Layout: BalancedLayout, + + Pin: false, + OnlyHash: false, + FsCache: false, + NoCopy: false, + + Events: nil, + Silent: false, + Progress: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, cid.Prefix{}, err + } + } + + // nocopy -> rawblocks + if options.NoCopy && !options.RawLeaves { + // fixed? + if options.RawLeavesSet { + return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") + } + + // No, satisfy mandatory constraint. + options.RawLeaves = true + } + + // (hash != "sha2-256") -> CIDv1 + if options.MhType != mh.SHA2_256 { + switch options.CidVersion { + case 0: + return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256") + case 1, -1: + options.CidVersion = 1 + default: + return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion) + } + } else { + if options.CidVersion < 0 { + // Default to CIDv0 + options.CidVersion = 0 + } + } + + // cidV1 -> raw blocks (by default) + if options.CidVersion > 0 && !options.RawLeavesSet { + options.RawLeaves = true + } + + prefix, err := dag.PrefixForCidVersion(options.CidVersion) + if err != nil { + return nil, cid.Prefix{}, err + } + + prefix.MhType = options.MhType + prefix.MhLength = -1 + + return options, prefix, nil +} + +func UnixfsLsOptions(opts ...UnixfsLsOption) (*UnixfsLsSettings, error) { + options := &UnixfsLsSettings{ + ResolveChildren: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type unixfsOpts struct{} + +var Unixfs unixfsOpts + +// CidVersion specifies which CID version to use. Defaults to 0 unless an option +// that depends on CIDv1 is passed. +func (unixfsOpts) CidVersion(version int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.CidVersion = version + return nil + } +} + +// Hash function to use. Implies CIDv1 if not set to sha2-256 (default). +// +// Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go +func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.MhType = mhtype + return nil + } +} + +// RawLeaves specifies whether to use raw blocks for leaves (data nodes with no +// links) instead of wrapping them with unixfs structures. +func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.RawLeaves = enable + settings.RawLeavesSet = true + return nil + } +} + +// Inline tells the adder to inline small blocks into CIDs +func (unixfsOpts) Inline(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Inline = enable + return nil + } +} + +// InlineLimit sets the amount of bytes below which blocks will be encoded +// directly into CID instead of being stored and addressed by it's hash. +// Specifying this option won't enable block inlining. For that use `Inline` +// option. Default: 32 bytes +// +// Note that while there is no hard limit on the number of bytes, it should be +// kept at a reasonably low value, such as 64; implementations may choose to +// reject anything larger. +func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.InlineLimit = limit + return nil + } +} + +// Chunker specifies settings for the chunking algorithm to use. +// +// Default: size-262144, formats: +// size-[bytes] - Simple chunker splitting data into blocks of n bytes +// rabin-[min]-[avg]-[max] - Rabin chunker +func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Chunker = chunker + return nil + } +} + +// Layout tells the adder how to balance data between leaves. +// options.BalancedLayout is the default, it's optimized for static seekable +// files. +// options.TrickleLayout is optimized for streaming data, +func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Layout = layout + return nil + } +} + +// Pin tells the adder to pin the file root recursively after adding +func (unixfsOpts) Pin(pin bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Pin = pin + return nil + } +} + +// HashOnly will make the adder calculate data hash without storing it in the +// blockstore or announcing it to the network +func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.OnlyHash = hashOnly + return nil + } +} + +// Events specifies channel which will be used to report events about ongoing +// Add operation. +// +// Note that if this channel blocks it may slowdown the adder +func (unixfsOpts) Events(sink chan<- interface{}) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Events = sink + return nil + } +} + +// Silent reduces event output +func (unixfsOpts) Silent(silent bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Silent = silent + return nil + } +} + +// Progress tells the adder whether to enable progress events +func (unixfsOpts) Progress(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Progress = enable + return nil + } +} + +// FsCache tells the adder to check the filestore for pre-existing blocks +// +// Experimental +func (unixfsOpts) FsCache(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.FsCache = enable + return nil + } +} + +// NoCopy tells the adder to add the files using filestore. Implies RawLeaves. +// +// Experimental +func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.NoCopy = enable + return nil + } +} + +func (unixfsOpts) ResolveChildren(resolve bool) UnixfsLsOption { + return func(settings *UnixfsLsSettings) error { + settings.ResolveChildren = resolve + return nil + } +} + +func (unixfsOpts) UseCumulativeSize(use bool) UnixfsLsOption { + return func(settings *UnixfsLsSettings) error { + settings.UseCumulativeSize = use + return nil + } +} diff --git a/core/coreiface/path/path.go b/core/coreiface/path/path.go new file mode 100644 index 000000000..c26b8692b --- /dev/null +++ b/core/coreiface/path/path.go @@ -0,0 +1,199 @@ +package path + +import ( + "strings" + + ipfspath "github.com/ipfs/boxo/path" + cid "github.com/ipfs/go-cid" +) + +// Path is a generic wrapper for paths used in the API. A path can be resolved +// to a CID using one of Resolve functions in the API. +// +// Paths must be prefixed with a valid prefix: +// +// * /ipfs - Immutable unixfs path (files) +// * /ipld - Immutable ipld path (data) +// * /ipns - Mutable names. Usually resolves to one of the immutable paths +// TODO: /local (MFS) +type Path interface { + // String returns the path as a string. + String() string + + // Namespace returns the first component of the path. + // + // For example path "/ipfs/QmHash", calling Namespace() will return "ipfs" + // + // Calling this method on invalid paths (IsValid() != nil) will result in + // empty string + Namespace() string + + // Mutable returns false if the data pointed to by this path in guaranteed + // to not change. + // + // Note that resolved mutable path can be immutable. + Mutable() bool + + // IsValid checks if this path is a valid ipfs Path, returning nil iff it is + // valid + IsValid() error +} + +// Resolved is a path which was resolved to the last resolvable node. +// ResolvedPaths are guaranteed to return nil from `IsValid` +type Resolved interface { + // Cid returns the CID of the node referenced by the path. Remainder of the + // path is guaranteed to be within the node. + // + // Examples: + // If you have 3 linked objects: QmRoot -> A -> B: + // + // cidB := {"foo": {"bar": 42 }} + // cidA := {"B": {"/": cidB }} + // cidRoot := {"A": {"/": cidA }} + // + // And resolve paths: + // + // * "/ipfs/${cidRoot}" + // * Calling Cid() will return `cidRoot` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `` + // + // * "/ipfs/${cidRoot}/A" + // * Calling Cid() will return `cidA` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `` + // + // * "/ipfs/${cidRoot}/A/B/foo" + // * Calling Cid() will return `cidB` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `foo` + // + // * "/ipfs/${cidRoot}/A/B/foo/bar" + // * Calling Cid() will return `cidB` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `foo/bar` + Cid() cid.Cid + + // Root returns the CID of the root object of the path + // + // Example: + // If you have 3 linked objects: QmRoot -> A -> B, and resolve path + // "/ipfs/QmRoot/A/B", the Root method will return the CID of object QmRoot + // + // For more examples see the documentation of Cid() method + Root() cid.Cid + + // Remainder returns unresolved part of the path + // + // Example: + // If you have 2 linked objects: QmRoot -> A, where A is a CBOR node + // containing the following data: + // + // {"foo": {"bar": 42 }} + // + // When resolving "/ipld/QmRoot/A/foo/bar", Remainder will return "foo/bar" + // + // For more examples see the documentation of Cid() method + Remainder() string + + Path +} + +// path implements coreiface.Path +type path struct { + path string +} + +// resolvedPath implements coreiface.resolvedPath +type resolvedPath struct { + path + cid cid.Cid + root cid.Cid + remainder string +} + +// Join appends provided segments to the base path +func Join(base Path, a ...string) Path { + s := strings.Join(append([]string{base.String()}, a...), "/") + return &path{path: s} +} + +// IpfsPath creates new /ipfs path from the provided CID +func IpfsPath(c cid.Cid) Resolved { + return &resolvedPath{ + path: path{"/ipfs/" + c.String()}, + cid: c, + root: c, + remainder: "", + } +} + +// IpldPath creates new /ipld path from the provided CID +func IpldPath(c cid.Cid) Resolved { + return &resolvedPath{ + path: path{"/ipld/" + c.String()}, + cid: c, + root: c, + remainder: "", + } +} + +// New parses string path to a Path +func New(p string) Path { + if pp, err := ipfspath.ParsePath(p); err == nil { + p = pp.String() + } + + return &path{path: p} +} + +// NewResolvedPath creates new Resolved path. This function performs no checks +// and is intended to be used by resolver implementations. Incorrect inputs may +// cause panics. Handle with care. +func NewResolvedPath(ipath ipfspath.Path, c cid.Cid, root cid.Cid, remainder string) Resolved { + return &resolvedPath{ + path: path{ipath.String()}, + cid: c, + root: root, + remainder: remainder, + } +} + +func (p *path) String() string { + return p.path +} + +func (p *path) Namespace() string { + ip, err := ipfspath.ParsePath(p.path) + if err != nil { + return "" + } + + if len(ip.Segments()) < 1 { + panic("path without namespace") // this shouldn't happen under any scenario + } + return ip.Segments()[0] +} + +func (p *path) Mutable() bool { + // TODO: MFS: check for /local + return p.Namespace() == "ipns" +} + +func (p *path) IsValid() error { + _, err := ipfspath.ParsePath(p.path) + return err +} + +func (p *resolvedPath) Cid() cid.Cid { + return p.cid +} + +func (p *resolvedPath) Root() cid.Cid { + return p.root +} + +func (p *resolvedPath) Remainder() string { + return p.remainder +} diff --git a/core/coreiface/pin.go b/core/coreiface/pin.go new file mode 100644 index 000000000..ba5df5354 --- /dev/null +++ b/core/coreiface/pin.go @@ -0,0 +1,63 @@ +package iface + +import ( + "context" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" +) + +// Pin holds information about pinned resource +type Pin interface { + // Path to the pinned object + Path() path.Resolved + + // Type of the pin + Type() string + + // if not nil, an error happened. Everything else should be ignored. + Err() error +} + +// PinStatus holds information about pin health +type PinStatus interface { + // Ok indicates whether the pin has been verified to be correct + Ok() bool + + // BadNodes returns any bad (usually missing) nodes from the pin + BadNodes() []BadPinNode +} + +// BadPinNode is a node that has been marked as bad by Pin.Verify +type BadPinNode interface { + // Path is the path of the node + Path() path.Resolved + + // Err is the reason why the node has been marked as bad + Err() error +} + +// PinAPI specifies the interface to pining +type PinAPI interface { + // Add creates new pin, be default recursive - pinning the whole referenced + // tree + Add(context.Context, path.Path, ...options.PinAddOption) error + + // Ls returns list of pinned objects on this node + Ls(context.Context, ...options.PinLsOption) (<-chan Pin, error) + + // IsPinned returns whether or not the given cid is pinned + // and an explanation of why its pinned + IsPinned(context.Context, path.Path, ...options.PinIsPinnedOption) (string, bool, error) + + // Rm removes pin for object specified by the path + Rm(context.Context, path.Path, ...options.PinRmOption) error + + // Update changes one pin to another, skipping checks for matching paths in + // the old tree + Update(ctx context.Context, from path.Path, to path.Path, opts ...options.PinUpdateOption) error + + // Verify verifies the integrity of pinned objects + Verify(context.Context) (<-chan PinStatus, error) +} diff --git a/core/coreiface/pubsub.go b/core/coreiface/pubsub.go new file mode 100644 index 000000000..bbd1da4ec --- /dev/null +++ b/core/coreiface/pubsub.go @@ -0,0 +1,48 @@ +package iface + +import ( + "context" + "io" + + "github.com/ipfs/boxo/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// PubSubSubscription is an active PubSub subscription +type PubSubSubscription interface { + io.Closer + + // Next return the next incoming message + Next(context.Context) (PubSubMessage, error) +} + +// PubSubMessage is a single PubSub message +type PubSubMessage interface { + // From returns id of a peer from which the message has arrived + From() peer.ID + + // Data returns the message body + Data() []byte + + // Seq returns message identifier + Seq() []byte + + // Topics returns list of topics this message was set to + Topics() []string +} + +// PubSubAPI specifies the interface to PubSub +type PubSubAPI interface { + // Ls lists subscribed topics by name + Ls(context.Context) ([]string, error) + + // Peers list peers we are currently pubsubbing with + Peers(context.Context, ...options.PubSubPeersOption) ([]peer.ID, error) + + // Publish a message to a given pubsub topic + Publish(context.Context, string, []byte) error + + // Subscribe to messages on a given topic + Subscribe(context.Context, string, ...options.PubSubSubscribeOption) (PubSubSubscription, error) +} diff --git a/core/coreiface/routing.go b/core/coreiface/routing.go new file mode 100644 index 000000000..a28ceb9e7 --- /dev/null +++ b/core/coreiface/routing.go @@ -0,0 +1,14 @@ +package iface + +import ( + "context" +) + +// RoutingAPI specifies the interface to the routing layer. +type RoutingAPI interface { + // Get retrieves the best value for a given key + Get(context.Context, string) ([]byte, error) + + // Put sets a value for a given key + Put(ctx context.Context, key string, value []byte) error +} diff --git a/core/coreiface/swarm.go b/core/coreiface/swarm.go new file mode 100644 index 000000000..9aa5466ba --- /dev/null +++ b/core/coreiface/swarm.go @@ -0,0 +1,57 @@ +package iface + +import ( + "context" + "errors" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + + ma "github.com/multiformats/go-multiaddr" +) + +var ( + ErrNotConnected = errors.New("not connected") + ErrConnNotFound = errors.New("conn not found") +) + +// ConnectionInfo contains information about a peer +type ConnectionInfo interface { + // ID returns PeerID + ID() peer.ID + + // Address returns the multiaddress via which we are connected with the peer + Address() ma.Multiaddr + + // Direction returns which way the connection was established + Direction() network.Direction + + // Latency returns last known round trip time to the peer + Latency() (time.Duration, error) + + // Streams returns list of streams established with the peer + Streams() ([]protocol.ID, error) +} + +// SwarmAPI specifies the interface to libp2p swarm +type SwarmAPI interface { + // Connect to a given peer + Connect(context.Context, peer.AddrInfo) error + + // Disconnect from a given address + Disconnect(context.Context, ma.Multiaddr) error + + // Peers returns the list of peers we are connected to + Peers(context.Context) ([]ConnectionInfo, error) + + // KnownAddrs returns the list of all addresses this node is aware of + KnownAddrs(context.Context) (map[peer.ID][]ma.Multiaddr, error) + + // LocalAddrs returns the list of announced listening addresses + LocalAddrs(context.Context) ([]ma.Multiaddr, error) + + // ListenAddrs returns the list of all listening addresses + ListenAddrs(context.Context) ([]ma.Multiaddr, error) +} diff --git a/core/coreiface/tests/api.go b/core/coreiface/tests/api.go new file mode 100644 index 000000000..497ef9d27 --- /dev/null +++ b/core/coreiface/tests/api.go @@ -0,0 +1,97 @@ +package tests + +import ( + "context" + "errors" + "testing" + "time" + + coreiface "github.com/ipfs/boxo/coreiface" +) + +var errAPINotImplemented = errors.New("api not implemented") + +func (tp *TestSuite) makeAPI(ctx context.Context) (coreiface.CoreAPI, error) { + api, err := tp.MakeAPISwarm(ctx, false, 1) + if err != nil { + return nil, err + } + + return api[0], nil +} + +type Provider interface { + // Make creates n nodes. fullIdentity set to false can be ignored + MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) +} + +func (tp *TestSuite) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) { + if tp.apis != nil { + tp.apis <- 1 + go func() { + <-ctx.Done() + tp.apis <- -1 + }() + } + + return tp.Provider.MakeAPISwarm(ctx, fullIdentity, n) +} + +type TestSuite struct { + Provider + + apis chan int +} + +func TestApi(p Provider) func(t *testing.T) { + running := 1 + apis := make(chan int) + zeroRunning := make(chan struct{}) + go func() { + for i := range apis { + running += i + if running < 1 { + close(zeroRunning) + return + } + } + }() + + tp := &TestSuite{Provider: p, apis: apis} + + return func(t *testing.T) { + t.Run("Block", tp.TestBlock) + t.Run("Dag", tp.TestDag) + t.Run("Dht", tp.TestDht) + t.Run("Key", tp.TestKey) + t.Run("Name", tp.TestName) + t.Run("Object", tp.TestObject) + t.Run("Path", tp.TestPath) + t.Run("Pin", tp.TestPin) + t.Run("PubSub", tp.TestPubSub) + t.Run("Routing", tp.TestRouting) + t.Run("Unixfs", tp.TestUnixfs) + + apis <- -1 + t.Run("TestsCancelCtx", func(t *testing.T) { + select { + case <-zeroRunning: + case <-time.After(time.Second): + t.Errorf("%d test swarms(s) not closed", running) + } + }) + } +} + +func (tp *TestSuite) hasApi(t *testing.T, tf func(coreiface.CoreAPI) error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if err := tf(api); err != nil { + t.Fatal(api) + } +} diff --git a/core/coreiface/tests/block.go b/core/coreiface/tests/block.go new file mode 100644 index 000000000..c884f3e82 --- /dev/null +++ b/core/coreiface/tests/block.go @@ -0,0 +1,354 @@ +package tests + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + coreiface "github.com/ipfs/boxo/coreiface" + opt "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + ipld "github.com/ipfs/go-ipld-format" + + mh "github.com/multiformats/go-multihash" +) + +var ( + pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb + pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb + rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes + cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor + cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512 +) + +// dag-pb +func pbBlock() io.Reader { + return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6}) +} + +// dag-cbor +func cborBlock() io.Reader { + return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111}) +} + +func (tp *TestSuite) TestBlock(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Block() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut) + t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb) + t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor) + t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor) + t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb) + t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0) + t.Run("TestBlockPutHash", tp.TestBlockPutHash) + t.Run("TestBlockGet", tp.TestBlockGet) + t.Run("TestBlockRm", tp.TestBlockRm) + t.Run("TestBlockStat", tp.TestBlockStat) + t.Run("TestBlockPin", tp.TestBlockPin) +} + +// when no opts are passed, produced CID has 'raw' codec +func (tp *TestSuite) TestBlockPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock()) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != rawCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm 'cbor' gets fixed to 'dag-cbor' +func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, cborBlock(), opt.Block.Format("cbor")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm 'protobuf' got fixed to 'dag-pb' +func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec) +func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCidV0 { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutHash(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put( + ctx, + cborBlock(), + opt.Block.Hash(mh.KECCAK_512, -1), + opt.Block.CidCodec("dag-cbor"), + ) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborKCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Block().Get(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + d, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } + + p := path.New("/ipfs/" + res.Path().Cid().String()) + + rp, err := api.ResolvePath(ctx, p) + if err != nil { + t.Fatal(err) + } + if rp.Cid().String() != res.Path().Cid().String() { + t.Error("paths didn't match") + } +} + +func (tp *TestSuite) TestBlockRm(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Block().Get(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + d, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } + + err = api.Block().Rm(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + _, err = api.Block().Get(ctx, res.Path()) + if err == nil { + t.Fatal("expected err to exist") + } + if !ipld.IsNotFound(err) { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res.Path()) + if err == nil { + t.Fatal("expected err to exist") + } + if !ipld.IsNotFound(err) { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res.Path(), opt.Block.Force(true)) + if err != nil { + t.Fatal(err) + } +} + +func (tp *TestSuite) TestBlockStat(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + stat, err := api.Block().Stat(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + if stat.Path().String() != res.Path().String() { + t.Error("paths don't match") + } + + if stat.Size() != len("Hello") { + t.Error("length doesn't match") + } +} + +func (tp *TestSuite) TestBlockPin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + if pins, err := api.Pin().Ls(ctx); err != nil || len(pins) != 0 { + t.Fatal("expected 0 pins") + } + + res, err := api.Block().Put( + ctx, + strings.NewReader(`Hello`), + opt.Block.Pin(true), + opt.Block.Format("raw"), + ) + if err != nil { + t.Fatal(err) + } + + pins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + if len(pins) != 1 { + t.Fatal("expected 1 pin") + } + if pins[0].Type() != "recursive" { + t.Error("expected a recursive pin") + } + if pins[0].Path().String() != res.Path().String() { + t.Error("pin path didn't match") + } +} diff --git a/core/coreiface/tests/dag.go b/core/coreiface/tests/dag.go new file mode 100644 index 000000000..b4118e2cc --- /dev/null +++ b/core/coreiface/tests/dag.go @@ -0,0 +1,200 @@ +package tests + +import ( + "context" + "math" + gopath "path" + "strings" + "testing" + + path "github.com/ipfs/boxo/coreiface/path" + + coreiface "github.com/ipfs/boxo/coreiface" + + ipldcbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +func (tp *TestSuite) TestDag(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Dag() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPut", tp.TestPut) + t.Run("TestPutWithHash", tp.TestPutWithHash) + t.Run("TestPath", tp.TestDagPath) + t.Run("TestTree", tp.TestTree) + t.Run("TestBatch", tp.TestBatch) +} + +var ( + treeExpected = map[string]struct{}{ + "a": {}, + "b": {}, + "c": {}, + "c/d": {}, + "c/e": {}, + } +) + +func (tp *TestSuite) TestPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } +} + +func (tp *TestSuite) TestPutWithHash(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), mh.SHA3_256, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyrmifu7haikttpqqgc5ewvmp76z3z4ebp7h2ph4memw7dq4nt6btmxny" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } +} + +func (tp *TestSuite) TestDagPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + snd, err := ipldcbor.FromJSON(strings.NewReader(`"foo"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, snd) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+snd.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + p := path.New(gopath.Join(nd.Cid().String(), "lnk")) + + rp, err := api.ResolvePath(ctx, p) + if err != nil { + t.Fatal(err) + } + + ndd, err := api.Dag().Get(ctx, rp.Cid()) + if err != nil { + t.Fatal(err) + } + + if ndd.Cid().String() != snd.Cid().String() { + t.Errorf("got unexpected cid %s, expected %s", ndd.Cid().String(), snd.Cid().String()) + } +} + +func (tp *TestSuite) TestTree(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"a": 123, "b": "foo", "c": {"d": 321, "e": 111}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + res, err := api.Dag().Get(ctx, nd.Cid()) + if err != nil { + t.Fatal(err) + } + + lst := res.Tree("", -1) + if len(lst) != len(treeExpected) { + t.Errorf("tree length of %d doesn't match expected %d", len(lst), len(treeExpected)) + } + + for _, ent := range lst { + if _, ok := treeExpected[ent]; !ok { + t.Errorf("unexpected tree entry %s", ent) + } + } +} + +func (tp *TestSuite) TestBatch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } + + _, err = api.Dag().Get(ctx, nd.Cid()) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{nd}); err != nil { + t.Fatal(err) + } + + _, err = api.Dag().Get(ctx, nd.Cid()) + if err != nil { + t.Fatal(err) + } +} diff --git a/core/coreiface/tests/dht.go b/core/coreiface/tests/dht.go new file mode 100644 index 000000000..fb3f6d1a0 --- /dev/null +++ b/core/coreiface/tests/dht.go @@ -0,0 +1,166 @@ +package tests + +import ( + "context" + "io" + "testing" + "time" + + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/options" +) + +func (tp *TestSuite) TestDht(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Dht() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestDhtFindPeer", tp.TestDhtFindPeer) + t.Run("TestDhtFindProviders", tp.TestDhtFindProviders) + t.Run("TestDhtProvide", tp.TestDhtProvide) +} + +func (tp *TestSuite) TestDhtFindPeer(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + laddrs0, err := apis[0].Swarm().LocalAddrs(ctx) + if err != nil { + t.Fatal(err) + } + if len(laddrs0) != 1 { + t.Fatal("unexpected number of local addrs") + } + + time.Sleep(3 * time.Second) + + pi, err := apis[2].Dht().FindPeer(ctx, self0.ID()) + if err != nil { + t.Fatal(err) + } + + if pi.Addrs[0].String() != laddrs0[0].String() { + t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) + } + + self2, err := apis[2].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + pi, err = apis[1].Dht().FindPeer(ctx, self2.ID()) + if err != nil { + t.Fatal(err) + } + + laddrs2, err := apis[2].Swarm().LocalAddrs(ctx) + if err != nil { + t.Fatal(err) + } + if len(laddrs2) != 1 { + t.Fatal("unexpected number of local addrs") + } + + if pi.Addrs[0].String() != laddrs2[0].String() { + t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) + } +} + +func (tp *TestSuite) TestDhtFindProviders(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + p, err := addTestObject(ctx, apis[0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + + out, err := apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + provider := <-out + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if provider.ID.String() != self0.ID().String() { + t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) + } +} + +func (tp *TestSuite) TestDhtProvide(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + off0, err := apis[0].WithOptions(options.Api.Offline(true)) + if err != nil { + t.Fatal(err) + } + + s, err := off0.Block().Put(ctx, &io.LimitedReader{R: rnd, N: 4092}) + if err != nil { + t.Fatal(err) + } + + p := s.Path() + + time.Sleep(3 * time.Second) + + out, err := apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + _, ok := <-out + + if ok { + t.Fatal("did not expect to find any providers") + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + err = apis[0].Dht().Provide(ctx, p) + if err != nil { + t.Fatal(err) + } + + out, err = apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + provider := <-out + + if provider.ID.String() != self0.ID().String() { + t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) + } +} diff --git a/core/coreiface/tests/key.go b/core/coreiface/tests/key.go new file mode 100644 index 000000000..3a38c07ae --- /dev/null +++ b/core/coreiface/tests/key.go @@ -0,0 +1,538 @@ +package tests + +import ( + "context" + "strings" + "testing" + + iface "github.com/ipfs/boxo/coreiface" + opt "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/go-cid" + mbase "github.com/multiformats/go-multibase" +) + +func (tp *TestSuite) TestKey(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Key() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestListSelf", tp.TestListSelf) + t.Run("TestRenameSelf", tp.TestRenameSelf) + t.Run("TestRemoveSelf", tp.TestRemoveSelf) + t.Run("TestGenerate", tp.TestGenerate) + t.Run("TestGenerateSize", tp.TestGenerateSize) + t.Run("TestGenerateType", tp.TestGenerateType) + t.Run("TestGenerateExisting", tp.TestGenerateExisting) + t.Run("TestList", tp.TestList) + t.Run("TestRename", tp.TestRename) + t.Run("TestRenameToSelf", tp.TestRenameToSelf) + t.Run("TestRenameToSelfForce", tp.TestRenameToSelfForce) + t.Run("TestRenameOverwriteNoForce", tp.TestRenameOverwriteNoForce) + t.Run("TestRenameOverwrite", tp.TestRenameOverwrite) + t.Run("TestRenameSameNameNoForce", tp.TestRenameSameNameNoForce) + t.Run("TestRenameSameName", tp.TestRenameSameName) + t.Run("TestRemove", tp.TestRemove) +} + +func (tp *TestSuite) TestListSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + keys, err := api.Key().List(ctx) + if err != nil { + t.Fatalf("failed to list keys: %s", err) + return + } + + if len(keys) != 1 { + t.Fatalf("there should be 1 key (self), got %d", len(keys)) + return + } + + if keys[0].Name() != "self" { + t.Errorf("expected the key to be called 'self', got '%s'", keys[0].Name()) + } + + if keys[0].Path().String() != "/ipns/"+iface.FormatKeyID(self.ID()) { + t.Errorf("expected the key to have path '/ipns/%s', got '%s'", iface.FormatKeyID(self.ID()), keys[0].Path().String()) + } +} + +func (tp *TestSuite) TestRenameSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "self", "foo") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot rename key with name 'self'") { + t.Fatalf("expected error 'cannot rename key with name 'self'', got '%s'", err.Error()) + } + } + + _, _, err = api.Key().Rename(ctx, "self", "foo", opt.Key.Force(true)) + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot rename key with name 'self'") { + t.Fatalf("expected error 'cannot rename key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRemoveSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Remove(ctx, "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot remove key with name 'self'") { + t.Fatalf("expected error 'cannot remove key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestGenerate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "foo" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + verifyIPNSPath(t, k.Path().String()) +} + +func verifyIPNSPath(t *testing.T, p string) bool { + t.Helper() + if !strings.HasPrefix(p, "/ipns/") { + t.Errorf("path %q does not look like an IPNS path", p) + return false + } + k := p[len("/ipns/"):] + c, err := cid.Decode(k) + if err != nil { + t.Errorf("failed to decode IPNS key %q (%v)", k, err) + return false + } + b36, err := c.StringOfBase(mbase.Base36) + if err != nil { + t.Fatalf("cid cannot format itself in b36") + return false + } + if b36 != k { + t.Errorf("IPNS key is not base36") + } + return true +} + +func (tp *TestSuite) TestGenerateSize(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo", opt.Key.Size(2048)) + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "foo" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + verifyIPNSPath(t, k.Path().String()) +} + +func (tp *TestSuite) TestGenerateType(t *testing.T) { + t.Skip("disabled until libp2p/specs#111 is fixed") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "bar", opt.Key.Type(opt.Ed25519Key)) + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "bar" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + // Expected to be an inlined identity hash. + if !strings.HasPrefix(k.Path().String(), "/ipns/12") { + t.Errorf("expected the key to be prefixed with '/ipns/12', got '%s'", k.Path().String()) + } +} + +func (tp *TestSuite) TestGenerateExisting(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "foo") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "key with name 'foo' already exists") { + t.Fatalf("expected error 'key with name 'foo' already exists', got '%s'", err.Error()) + } + } + + _, err = api.Key().Generate(ctx, "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot create key with name 'self'") { + t.Fatalf("expected error 'cannot create key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestList(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + l, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 2 { + t.Fatalf("expected to get 2 keys, got %d", len(l)) + return + } + + if l[0].Name() != "self" { + t.Fatalf("expected key 0 to be called 'self', got '%s'", l[0].Name()) + return + } + + if l[1].Name() != "foo" { + t.Fatalf("expected key 1 to be called 'foo', got '%s'", l[1].Name()) + return + } + + verifyIPNSPath(t, l[0].Path().String()) + verifyIPNSPath(t, l[1].Path().String()) +} + +func (tp *TestSuite) TestRename(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "bar") + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "bar" { + t.Errorf("returned key should be called 'bar', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRenameToSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot overwrite key with name 'self'") { + t.Fatalf("expected error 'cannot overwrite key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameToSelfForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "self", opt.Key.Force(true)) + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot overwrite key with name 'self'") { + t.Fatalf("expected error 'cannot overwrite key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameOverwriteNoForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "bar") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "bar") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "key by that name already exists, refusing to overwrite") { + t.Fatalf("expected error 'key by that name already exists, refusing to overwrite', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameOverwrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + kfoo, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "bar") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "bar", opt.Key.Force(true)) + if err != nil { + t.Fatal(err) + return + } + + if !overwrote { + t.Error("overwrote should be true") + } + + if k.Name() != "bar" { + t.Errorf("returned key should be called 'bar', got '%s'", k.Name()) + } + + if k.Path().String() != kfoo.Path().String() { + t.Errorf("k and kfoo should have equal paths, '%s'!='%s'", k.Path().String(), kfoo.Path().String()) + } +} + +func (tp *TestSuite) TestRenameSameNameNoForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "foo") + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "foo" { + t.Errorf("returned key should be called 'foo', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRenameSameName(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "foo", opt.Key.Force(true)) + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "foo" { + t.Errorf("returned key should be called 'foo', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRemove(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + l, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 2 { + t.Fatalf("expected to get 2 keys, got %d", len(l)) + return + } + + p, err := api.Key().Remove(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + if k.Path().String() != p.Path().String() { + t.Errorf("k and p should have equal paths, '%s'!='%s'", k.Path().String(), p.Path().String()) + } + + l, err = api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 1 { + t.Fatalf("expected to get 1 key, got %d", len(l)) + return + } + + if l[0].Name() != "self" { + t.Errorf("expected the key to be called 'self', got '%s'", l[0].Name()) + } +} diff --git a/core/coreiface/tests/name.go b/core/coreiface/tests/name.go new file mode 100644 index 000000000..a67876cba --- /dev/null +++ b/core/coreiface/tests/name.go @@ -0,0 +1,274 @@ +package tests + +import ( + "context" + "io" + "math/rand" + gopath "path" + "testing" + "time" + + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/files" + + coreiface "github.com/ipfs/boxo/coreiface" + opt "github.com/ipfs/boxo/coreiface/options" +) + +func (tp *TestSuite) TestName(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Name() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPublishResolve", tp.TestPublishResolve) + t.Run("TestBasicPublishResolveKey", tp.TestBasicPublishResolveKey) + t.Run("TestBasicPublishResolveTimeout", tp.TestBasicPublishResolveTimeout) +} + +var rnd = rand.New(rand.NewSource(0x62796532303137)) + +func addTestObject(ctx context.Context, api coreiface.CoreAPI) (path.Path, error) { + return api.Unixfs().Add(ctx, files.NewReaderFile(&io.LimitedReader{R: rnd, N: 4092})) +} + +func appendPath(p path.Path, sub string) path.Path { + return path.New(gopath.Join(p.String(), sub)) +} + +func (tp *TestSuite) TestPublishResolve(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := func() (coreiface.CoreAPI, path.Path) { + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + return nil, nil + } + api := apis[0] + + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + return nil, nil + } + return api, p + } + run := func(t *testing.T, ropts []opt.NameResolveOption) { + t.Run("basic", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name(), ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()) + } + }) + + t.Run("publishPath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, appendPath(p, "/test")) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name(), ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/test") + } + }) + + t.Run("revolvePath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()+"/test", ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/test") + } + }) + + t.Run("publishRevolvePath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, appendPath(p, "/a")) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String()+"/a" { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()+"/b", ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/a/b" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/a/b") + } + }) + } + + t.Run("default", func(t *testing.T) { + run(t, []opt.NameResolveOption{}) + }) + + t.Run("nocache", func(t *testing.T) { + run(t, []opt.NameResolveOption{opt.Name.Cache(false)}) + }) +} + +func (tp *TestSuite) TestBasicPublishResolveKey(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + api := apis[0] + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + } + + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + e, err := api.Name().Publish(ctx, p, opt.Name.Key(k.Name())) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKey(k) { + t.Errorf("expected e.Name to equal %s, got '%s'", e.Name(), coreiface.FormatKey(k)) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()) + } +} + +func (tp *TestSuite) TestBasicPublishResolveTimeout(t *testing.T) { + t.Skip("ValidTime doesn't appear to work at this time resolution") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + api := apis[0] + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + e, err := api.Name().Publish(ctx, p, opt.Name.ValidTime(time.Millisecond*100)) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + time.Sleep(time.Second) + + _, err = api.Name().Resolve(ctx, e.Name()) + if err == nil { + t.Fatal("Expected an error") + } +} + +//TODO: When swarm api is created, add multinode tests diff --git a/core/coreiface/tests/object.go b/core/coreiface/tests/object.go new file mode 100644 index 000000000..8e8f52b3d --- /dev/null +++ b/core/coreiface/tests/object.go @@ -0,0 +1,467 @@ +package tests + +import ( + "bytes" + "context" + "encoding/hex" + "io" + "strings" + "testing" + + iface "github.com/ipfs/boxo/coreiface" + opt "github.com/ipfs/boxo/coreiface/options" +) + +func (tp *TestSuite) TestObject(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Object() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestNew", tp.TestNew) + t.Run("TestObjectPut", tp.TestObjectPut) + t.Run("TestObjectGet", tp.TestObjectGet) + t.Run("TestObjectData", tp.TestObjectData) + t.Run("TestObjectLinks", tp.TestObjectLinks) + t.Run("TestObjectStat", tp.TestObjectStat) + t.Run("TestObjectAddLink", tp.TestObjectAddLink) + t.Run("TestObjectAddLinkCreate", tp.TestObjectAddLinkCreate) + t.Run("TestObjectRmLink", tp.TestObjectRmLink) + t.Run("TestObjectAddData", tp.TestObjectAddData) + t.Run("TestObjectSetData", tp.TestObjectSetData) + t.Run("TestDiffTest", tp.TestDiffTest) +} + +func (tp *TestSuite) TestNew(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + emptyNode, err := api.Object().New(ctx) + if err != nil { + t.Fatal(err) + } + + dirNode, err := api.Object().New(ctx, opt.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + if emptyNode.String() != "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" { + t.Errorf("Unexpected emptyNode path: %s", emptyNode.String()) + } + + if dirNode.String() != "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" { + t.Errorf("Unexpected dirNode path: %s", dirNode.String()) + } +} + +func (tp *TestSuite) TestObjectPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"YmFy"}`), opt.Object.DataType("base64")) //bar + if err != nil { + t.Fatal(err) + } + + pbBytes, err := hex.DecodeString("0a0362617a") + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().Put(ctx, bytes.NewReader(pbBytes), opt.Object.InputEnc("protobuf")) + if err != nil { + t.Fatal(err) + } + + if p1.String() != "/ipfs/QmQeGyS87nyijii7kFt1zbe4n2PsXTFimzsdxyE9qh9TST" { + t.Errorf("unexpected path: %s", p1.String()) + } + + if p2.String() != "/ipfs/QmNeYRbCibmaMMK6Du6ChfServcLqFvLJF76PzzF76SPrZ" { + t.Errorf("unexpected path: %s", p2.String()) + } + + if p3.String() != "/ipfs/QmZreR7M2t7bFXAdb1V5FtQhjk4t36GnrvueLJowJbQM9m" { + t.Errorf("unexpected path: %s", p3.String()) + } +} + +func (tp *TestSuite) TestObjectGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + nd, err := api.Object().Get(ctx, p1) + if err != nil { + t.Fatal(err) + } + + if string(nd.RawData()[len(nd.RawData())-3:]) != "foo" { + t.Fatal("got non-matching data") + } +} + +func (tp *TestSuite) TestObjectData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p1) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "foo" { + t.Fatal("got non-matching data") + } +} + +func (tp *TestSuite) TestObjectLinks(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`"}]}`)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if len(links) != 1 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Cid.String() != p1.Cid().String() { + t.Fatal("cids didn't batch") + } + + if links[0].Name != "bar" { + t.Fatal("unexpected link name") + } +} + +func (tp *TestSuite) TestObjectStat(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + stat, err := api.Object().Stat(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if stat.Cid.String() != p2.Cid().String() { + t.Error("unexpected stat.Cid") + } + + if stat.NumLinks != 1 { + t.Errorf("unexpected stat.NumLinks") + } + + if stat.BlockSize != 51 { + t.Error("unexpected stat.BlockSize") + } + + if stat.LinksSize != 47 { + t.Errorf("unexpected stat.LinksSize: %d", stat.LinksSize) + } + + if stat.DataSize != 4 { + t.Error("unexpected stat.DataSize") + } + + if stat.CumulativeSize != 54 { + t.Error("unexpected stat.DataSize") + } +} + +func (tp *TestSuite) TestObjectAddLink(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc", p2) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func (tp *TestSuite) TestObjectAddLinkCreate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + _, err = api.Object().AddLink(ctx, p2, "abc/d", p2) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "no link by that name") { + t.Fatalf("unexpected error: %s", err.Error()) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc/d", p2, opt.Object.Create(true)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func (tp *TestSuite) TestObjectRmLink(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().RmLink(ctx, p2, "bar") + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Errorf("unexpected number of links: %d", len(links)) + } +} + +func (tp *TestSuite) TestObjectAddData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().AppendData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "foobar" { + t.Error("unexpected data") + } +} + +func (tp *TestSuite) TestObjectSetData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().SetData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "bar" { + t.Error("unexpected data") + } +} + +func (tp *TestSuite) TestDiffTest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bar"}`)) + if err != nil { + t.Fatal(err) + } + + changes, err := api.Object().Diff(ctx, p1, p2) + if err != nil { + t.Fatal(err) + } + + if len(changes) != 1 { + t.Fatal("unexpected changes len") + } + + if changes[0].Type != iface.DiffMod { + t.Fatal("unexpected change type") + } + + if changes[0].Before.String() != p1.String() { + t.Fatal("unexpected before path") + } + + if changes[0].After.String() != p2.String() { + t.Fatal("unexpected before path") + } +} diff --git a/core/coreiface/tests/path.go b/core/coreiface/tests/path.go new file mode 100644 index 000000000..321c79d6e --- /dev/null +++ b/core/coreiface/tests/path.go @@ -0,0 +1,197 @@ +package tests + +import ( + "context" + "math" + "strings" + "testing" + + "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/coreiface/options" + + ipldcbor "github.com/ipfs/go-ipld-cbor" +) + +func (tp *TestSuite) TestPath(t *testing.T) { + t.Run("TestMutablePath", tp.TestMutablePath) + t.Run("TestPathRemainder", tp.TestPathRemainder) + t.Run("TestEmptyPathRemainder", tp.TestEmptyPathRemainder) + t.Run("TestInvalidPathRemainder", tp.TestInvalidPathRemainder) + t.Run("TestPathRoot", tp.TestPathRoot) + t.Run("TestPathJoin", tp.TestPathJoin) +} + +func (tp *TestSuite) TestMutablePath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + blk, err := api.Block().Put(ctx, strings.NewReader(`foo`)) + if err != nil { + t.Fatal(err) + } + + if blk.Path().Mutable() { + t.Error("expected /ipld path to be immutable") + } + + // get self /ipns path + + if api.Key() == nil { + t.Fatal(".Key not implemented") + } + + keys, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + } + + if !keys[0].Path().Mutable() { + t.Error("expected self /ipns path to be mutable") + } +} + +func (tp *TestSuite) TestPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp1, err := api.ResolvePath(ctx, path.New(nd.String()+"/foo/bar")) + if err != nil { + t.Fatal(err) + } + + if rp1.Remainder() != "foo/bar" { + t.Error("expected to get path remainder") + } +} + +func (tp *TestSuite) TestEmptyPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp1, err := api.ResolvePath(ctx, path.New(nd.Cid().String())) + if err != nil { + t.Fatal(err) + } + + if rp1.Remainder() != "" { + t.Error("expected the resolved path to not have a remainder") + } +} + +func (tp *TestSuite) TestInvalidPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + _, err = api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/bar/baz")) + if err == nil || !strings.Contains(err.Error(), `no link named "bar"`) { + t.Fatalf("unexpected error: %s", err) + } +} + +func (tp *TestSuite) TestPathRoot(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Block() == nil { + t.Fatal(".Block not implemented") + } + + blk, err := api.Block().Put(ctx, strings.NewReader(`foo`), options.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"/": "`+blk.Path().Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp, err := api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/foo")) + if err != nil { + t.Fatal(err) + } + + if rp.Root().String() != nd.Cid().String() { + t.Error("unexpected path root") + } + + if rp.Cid().String() != blk.Path().Cid().String() { + t.Error("unexpected path cid") + } +} + +func (tp *TestSuite) TestPathJoin(t *testing.T) { + p1 := path.New("/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz") + + if path.Join(p1, "foo").String() != "/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz/foo" { + t.Error("unexpected path") + } +} diff --git a/core/coreiface/tests/pin.go b/core/coreiface/tests/pin.go new file mode 100644 index 000000000..ac90d097e --- /dev/null +++ b/core/coreiface/tests/pin.go @@ -0,0 +1,601 @@ +package tests + +import ( + "context" + "math" + "strings" + "testing" + + iface "github.com/ipfs/boxo/coreiface" + opt "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" +) + +func (tp *TestSuite) TestPin(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Pin() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPinAdd", tp.TestPinAdd) + t.Run("TestPinSimple", tp.TestPinSimple) + t.Run("TestPinRecursive", tp.TestPinRecursive) + t.Run("TestPinLsIndirect", tp.TestPinLsIndirect) + t.Run("TestPinLsPrecedence", tp.TestPinLsPrecedence) + t.Run("TestPinIsPinned", tp.TestPinIsPinned) +} + +func (tp *TestSuite) TestPinAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, p) + if err != nil { + t.Fatal(err) + } +} + +func (tp *TestSuite) TestPinSimple(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, p) + if err != nil { + t.Fatal(err) + } + + list, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().Cid().String() != p.Cid().String() { + t.Error("paths don't match") + } + + if list[0].Type() != "recursive" { + t.Error("unexpected pin type") + } + + assertIsPinned(t, ctx, api, p, "recursive") + + err = api.Pin().Rm(ctx, p) + if err != nil { + t.Fatal(err) + } + + list, err = accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 0 { + t.Errorf("unexpected pin list len: %d", len(list)) + } +} + +func (tp *TestSuite) TestPinRecursive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p0, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Unixfs().Add(ctx, strFile("bar")()) + if err != nil { + t.Fatal(err) + } + + nd2, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p0.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + nd3, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p1.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{nd2, nd3}); err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(nd2.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(nd3.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + list, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 3 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().String() != path.IpldPath(nd3.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpfsPath(nd3.Cid()).String()) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().String() != path.IpldPath(nd2.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpldPath(nd2.Cid()).String()) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().Cid().String() != p0.Cid().String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().Cid().String(), p0.Cid().String()) + } + + res, err := api.Pin().Verify(ctx) + if err != nil { + t.Fatal(err) + } + n := 0 + for r := range res { + if !r.Ok() { + t.Error("expected pin to be ok") + } + n++ + } + + if n != 1 { + t.Errorf("unexpected verify result count: %d", n) + } + + //TODO: figure out a way to test verify without touching IpfsNode + /* + err = api.Block().Rm(ctx, p0, opt.Block.Force(true)) + if err != nil { + t.Fatal(err) + } + + res, err = api.Pin().Verify(ctx) + if err != nil { + t.Fatal(err) + } + n = 0 + for r := range res { + if r.Ok() { + t.Error("expected pin to not be ok") + } + + if len(r.BadNodes()) != 1 { + t.Fatalf("unexpected badNodes len") + } + + if r.BadNodes()[0].Path().Cid().String() != p0.Cid().String() { + t.Error("unexpected badNode path") + } + + if r.BadNodes()[0].Err().Error() != "merkledag: not found" { + t.Errorf("unexpected badNode error: %s", r.BadNodes()[0].Err().Error()) + } + n++ + } + + if n != 1 { + t.Errorf("unexpected verify result count: %d", n) + } + */ +} + +// TestPinLsIndirect verifies that indirect nodes are listed by pin ls even if a parent node is directly pinned +func (tp *TestSuite) TestPinLsIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foo") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) +} + +// TestPinLsPrecedence verifies the precedence of pins (recursive > direct > indirect) +func (tp *TestSuite) TestPinLsPrecedence(t *testing.T) { + // Testing precedence of recursive, direct and indirect pins + // Results should be recursive > indirect, direct > indirect, and recursive > direct + + t.Run("TestPinLsPredenceRecursiveIndirect", tp.TestPinLsPredenceRecursiveIndirect) + t.Run("TestPinLsPrecedenceDirectIndirect", tp.TestPinLsPrecedenceDirectIndirect) + t.Run("TestPinLsPrecedenceRecursiveDirect", tp.TestPinLsPrecedenceRecursiveDirect) +} + +func (tp *TestSuite) TestPinLsPredenceRecursiveIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test recursive > indirect + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive > indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinLsPrecedenceDirectIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test direct > indirect + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "direct > indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinLsPrecedenceRecursiveDirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test recursive > direct + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive + direct = error") + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err == nil { + t.Fatal("expected error directly pinning a recursively pinned node") + } + + assertPinTypes(t, ctx, api, []cidContainer{parent}, []cidContainer{}, []cidContainer{leaf}) + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinIsPinned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foofoo") + + assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) + assertNotPinned(t, ctx, api, path.IpldPath(parent.Cid())) + assertNotPinned(t, ctx, api, path.IpldPath(leaf.Cid())) + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(true)) + if err != nil { + t.Fatal(err) + } + + assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) + assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertIsPinned(t, ctx, api, path.IpldPath(grandparent.Cid()), "direct") + assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") +} + +type cidContainer interface { + Cid() cid.Cid +} + +func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, leafData string) (cidContainer, cidContainer, cidContainer) { + leaf, err := api.Unixfs().Add(ctx, strFile(leafData)()) + if err != nil { + t.Fatal(err) + } + + parent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+leaf.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + grandparent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+parent.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{parent, grandparent}); err != nil { + t.Fatal(err) + } + + return leaf, parent, grandparent +} + +func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recusive, direct, indirect []cidContainer) { + assertPinLsAllConsistency(t, ctx, api) + + list, err := accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, recusive...) + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, direct...) + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, indirect...) +} + +// assertPinCids verifies that the pins match the expected cids +func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { + t.Helper() + + if expected, actual := len(cids), len(pins); expected != actual { + t.Fatalf("expected pin list to have len %d, was %d", expected, actual) + } + + cSet := cid.NewSet() + for _, c := range cids { + cSet.Add(c.Cid()) + } + + valid := true + for _, p := range pins { + c := p.Path().Cid() + if cSet.Has(c) { + cSet.Remove(c) + } else { + valid = false + break + } + } + + valid = valid && cSet.Len() == 0 + + if !valid { + pinStrs := make([]string, len(pins)) + for i, p := range pins { + pinStrs[i] = p.Path().Cid().String() + } + pathStrs := make([]string, len(cids)) + for i, c := range cids { + pathStrs[i] = c.Cid().String() + } + t.Fatalf("expected: %s \nactual: %s", strings.Join(pathStrs, ", "), strings.Join(pinStrs, ", ")) + } +} + +// assertPinLsAllConsistency verifies that listing all pins gives the same result as listing the pin types individually +func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.CoreAPI) { + t.Helper() + allPins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + type pinTypeProps struct { + *cid.Set + opt.PinLsOption + } + + all, recursive, direct, indirect := cid.NewSet(), cid.NewSet(), cid.NewSet(), cid.NewSet() + typeMap := map[string]*pinTypeProps{ + "recursive": {recursive, opt.Pin.Ls.Recursive()}, + "direct": {direct, opt.Pin.Ls.Direct()}, + "indirect": {indirect, opt.Pin.Ls.Indirect()}, + } + + for _, p := range allPins { + if !all.Visit(p.Path().Cid()) { + t.Fatalf("pin ls returned the same cid multiple times") + } + + typeStr := p.Type() + if typeSet, ok := typeMap[p.Type()]; ok { + typeSet.Add(p.Path().Cid()) + } else { + t.Fatalf("unknown pin type: %s", typeStr) + } + } + + for typeStr, pinProps := range typeMap { + pins, err := accPins(api.Pin().Ls(ctx, pinProps.PinLsOption)) + if err != nil { + t.Fatal(err) + } + + if expected, actual := len(pins), pinProps.Set.Len(); expected != actual { + t.Fatalf("pin ls all has %d pins of type %s, but pin ls for the type has %d", expected, typeStr, actual) + } + + for _, p := range pins { + if pinType := p.Type(); pinType != typeStr { + t.Fatalf("returned wrong pin type: expected %s, got %s", typeStr, pinType) + } + + if c := p.Path().Cid(); !pinProps.Has(c) { + t.Fatalf("%s expected to be in pin ls all as type %s", c.String(), typeStr) + } + } + } +} + +func assertIsPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path, typeStr string) { + t.Helper() + withType, err := opt.Pin.IsPinned.Type(typeStr) + if err != nil { + t.Fatal("unhandled pin type") + } + + whyPinned, pinned, err := api.Pin().IsPinned(ctx, p, withType) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatalf("%s expected to be pinned with type %s", p, typeStr) + } + + switch typeStr { + case "recursive", "direct": + if typeStr != whyPinned { + t.Fatalf("reason for pinning expected to be %s for %s, got %s", typeStr, p, whyPinned) + } + case "indirect": + if whyPinned == "" { + t.Fatalf("expected to have a pin reason for %s", p) + } + } +} + +func assertNotPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path) { + t.Helper() + + _, pinned, err := api.Pin().IsPinned(ctx, p) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatalf("%s expected to not be pinned", p) + } +} + +func accPins(pins <-chan iface.Pin, err error) ([]iface.Pin, error) { + if err != nil { + return nil, err + } + + var result []iface.Pin + + for pin := range pins { + if pin.Err() != nil { + return nil, pin.Err() + } + result = append(result, pin) + } + + return result, nil +} diff --git a/core/coreiface/tests/pubsub.go b/core/coreiface/tests/pubsub.go new file mode 100644 index 000000000..446e0771a --- /dev/null +++ b/core/coreiface/tests/pubsub.go @@ -0,0 +1,136 @@ +package tests + +import ( + "context" + "testing" + "time" + + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/options" +) + +func (tp *TestSuite) TestPubSub(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.PubSub() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestBasicPubSub", tp.TestBasicPubSub) +} + +func (tp *TestSuite) TestBasicPubSub(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + sub, err := apis[0].PubSub().Subscribe(ctx, "testch") + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + err := apis[1].PubSub().Publish(ctx, "testch", []byte("hello world")) + switch err { + case nil: + case context.Canceled: + return + default: + t.Error(err) + cancel() + return + } + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } + }() + + // Wait for the sender to finish before we return. + // Otherwise, we can get random errors as publish fails. + defer func() { + cancel() + <-done + }() + + m, err := sub.Next(ctx) + if err != nil { + t.Fatal(err) + } + + if string(m.Data()) != "hello world" { + t.Errorf("got invalid data: %s", string(m.Data())) + } + + self1, err := apis[1].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if m.From() != self1.ID() { + t.Errorf("m.From didn't match") + } + + peers, err := apis[1].PubSub().Peers(ctx, options.PubSub.Topic("testch")) + if err != nil { + t.Fatal(err) + } + + if len(peers) != 1 { + t.Fatalf("got incorrect number of peers: %d", len(peers)) + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if peers[0] != self0.ID() { + t.Errorf("peer didn't match") + } + + peers, err = apis[1].PubSub().Peers(ctx, options.PubSub.Topic("nottestch")) + if err != nil { + t.Fatal(err) + } + + if len(peers) != 0 { + t.Fatalf("got incorrect number of peers: %d", len(peers)) + } + + topics, err := apis[0].PubSub().Ls(ctx) + if err != nil { + t.Fatal(err) + } + + if len(topics) != 1 { + t.Fatalf("got incorrect number of topics: %d", len(peers)) + } + + if topics[0] != "testch" { + t.Errorf("topic didn't match") + } + + topics, err = apis[1].PubSub().Ls(ctx) + if err != nil { + t.Fatal(err) + } + + if len(topics) != 0 { + t.Fatalf("got incorrect number of topics: %d", len(peers)) + } +} diff --git a/core/coreiface/tests/routing.go b/core/coreiface/tests/routing.go new file mode 100644 index 000000000..e1d8d1060 --- /dev/null +++ b/core/coreiface/tests/routing.go @@ -0,0 +1,92 @@ +package tests + +import ( + "context" + "testing" + "time" + + "github.com/gogo/protobuf/proto" + iface "github.com/ipfs/boxo/coreiface" + ipns_pb "github.com/ipfs/boxo/ipns/pb" +) + +func (tp *TestSuite) TestRouting(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Routing() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestRoutingGet", tp.TestRoutingGet) + t.Run("TestRoutingPut", tp.TestRoutingPut) +} + +func (tp *TestSuite) testRoutingPublishKey(t *testing.T, ctx context.Context, api iface.CoreAPI) iface.IpnsEntry { + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + entry, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + return entry +} + +func (tp *TestSuite) TestRoutingGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + // Node 1: publishes an IPNS name + ipnsEntry := tp.testRoutingPublishKey(t, ctx, apis[0]) + + // Node 2: retrieves the best value for the IPNS name. + data, err := apis[1].Routing().Get(ctx, "/ipns/"+ipnsEntry.Name()) + if err != nil { + t.Fatal(err) + } + + // Checks if values match. + var entry ipns_pb.IpnsEntry + err = proto.Unmarshal(data, &entry) + if err != nil { + t.Fatal(err) + } + + if string(entry.GetValue()) != ipnsEntry.Value().String() { + t.Fatalf("routing key has wrong value, expected %s, got %s", ipnsEntry.Value().String(), string(entry.GetValue())) + } +} + +func (tp *TestSuite) TestRoutingPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + // Create and publish IPNS entry. + ipnsEntry := tp.testRoutingPublishKey(t, ctx, apis[0]) + + // Get valid routing value. + data, err := apis[0].Routing().Get(ctx, "/ipns/"+ipnsEntry.Name()) + if err != nil { + t.Fatal(err) + } + + // Put routing value. + err = apis[1].Routing().Put(ctx, "/ipns/"+ipnsEntry.Name(), data) + if err != nil { + t.Fatal(err) + } +} diff --git a/core/coreiface/tests/unixfs.go b/core/coreiface/tests/unixfs.go new file mode 100644 index 000000000..cca42bb28 --- /dev/null +++ b/core/coreiface/tests/unixfs.go @@ -0,0 +1,1080 @@ +package tests + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "math" + "math/rand" + "os" + "strconv" + "strings" + "sync" + "testing" + + "github.com/ipfs/boxo/coreiface/path" + + coreiface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/options" + + "github.com/ipfs/boxo/files" + mdag "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +func (tp *TestSuite) TestUnixfs(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Unixfs() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestAdd", tp.TestAdd) + t.Run("TestAddPinned", tp.TestAddPinned) + t.Run("TestAddHashOnly", tp.TestAddHashOnly) + t.Run("TestGetEmptyFile", tp.TestGetEmptyFile) + t.Run("TestGetDir", tp.TestGetDir) + t.Run("TestGetNonUnixfs", tp.TestGetNonUnixfs) + t.Run("TestLs", tp.TestLs) + t.Run("TestEntriesExpired", tp.TestEntriesExpired) + t.Run("TestLsEmptyDir", tp.TestLsEmptyDir) + t.Run("TestLsNonUnixfs", tp.TestLsNonUnixfs) + t.Run("TestAddCloses", tp.TestAddCloses) + t.Run("TestGetSeek", tp.TestGetSeek) + t.Run("TestGetReadAt", tp.TestGetReadAt) +} + +// `echo -n 'hello, world!' | ipfs add` +var hello = "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" +var helloStr = "hello, world!" + +// `echo -n | ipfs add` +var emptyFile = "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" + +func strFile(data string) func() files.Node { + return func() files.Node { + return files.NewBytesFile([]byte(data)) + } +} + +func twoLevelDir() func() files.Node { + return func() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + "abc": files.NewMapDirectory(map[string]files.Node{ + "def": files.NewBytesFile([]byte("world")), + }), + + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) + } +} + +func flatDir() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) +} + +func wrapped(names ...string) func(f files.Node) files.Node { + return func(f files.Node) files.Node { + for i := range names { + f = files.NewMapDirectory(map[string]files.Node{ + names[len(names)-i-1]: f, + }) + } + return f + } +} + +func (tp *TestSuite) TestAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p := func(h string) path.Resolved { + c, err := cid.Parse(h) + if err != nil { + t.Fatal(err) + } + return path.IpfsPath(c) + } + + rf, err := os.CreateTemp(os.TempDir(), "unixfs-add-real") + if err != nil { + t.Fatal(err) + } + rfp := rf.Name() + + if _, err := rf.Write([]byte(helloStr)); err != nil { + t.Fatal(err) + } + + stat, err := rf.Stat() + if err != nil { + t.Fatal(err) + } + + if err := rf.Close(); err != nil { + t.Fatal(err) + } + defer os.Remove(rfp) + + realFile := func() files.Node { + n, err := files.NewReaderPathFile(rfp, io.NopCloser(strings.NewReader(helloStr)), stat) + if err != nil { + t.Fatal(err) + } + return n + } + + cases := []struct { + name string + data func() files.Node + expect func(files.Node) files.Node + + apiOpts []options.ApiOption + + path string + err string + + wrap string + + events []coreiface.AddEvent + + opts []options.UnixfsAddOption + }{ + // Simple cases + { + name: "simpleAdd", + data: strFile(helloStr), + path: hello, + opts: []options.UnixfsAddOption{}, + }, + { + name: "addEmpty", + data: strFile(""), + path: emptyFile, + }, + // CIDv1 version / rawLeaves + { + name: "addCidV1", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, + }, + { + name: "addCidV1NoLeaves", + data: strFile(helloStr), + path: "/ipfs/bafybeibhbcn7k7o2m6xsqkrlfiokod3nxwe47viteynhruh6uqx7hvkjfu", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(false)}, + }, + // Non sha256 hash vs CID + { + name: "addCidSha3", + data: strFile(helloStr), + path: "/ipfs/bafkrmichjflejeh6aren53o7pig7zk3m3vxqcoc2i5dv326k3x6obh7jry", + opts: []options.UnixfsAddOption{options.Unixfs.Hash(mh.SHA3_256)}, + }, + { + name: "addCidSha3Cid0", + data: strFile(helloStr), + err: "CIDv0 only supports sha2-256", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, + }, + // Inline + { + name: "addInline", + data: strFile(helloStr), + path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", + opts: []options.UnixfsAddOption{options.Unixfs.Inline(true)}, + }, + { + name: "addInlineLimit", + data: strFile(helloStr), + path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true)}, + }, + { + name: "addInlineZero", + data: strFile(""), + path: "/ipfs/bafkqaaa", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(0), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, + }, + { //TODO: after coreapi add is used in `ipfs add`, consider making this default for inline + name: "addInlineRaw", + data: strFile(helloStr), + path: "/ipfs/bafkqadlimvwgy3zmeb3w64tmmqqq", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, + }, + // Chunker / Layout + { + name: "addChunks", + data: strFile(strings.Repeat("aoeuidhtns", 200)), + path: "/ipfs/QmRo11d4QJrST47aaiGVJYwPhoNA4ihRpJ5WaxBWjWDwbX", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4")}, + }, + { + name: "addChunksTrickle", + data: strFile(strings.Repeat("aoeuidhtns", 200)), + path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLayout)}, + }, + // Local + { + name: "addLocal", // better cases in sharness + data: strFile(helloStr), + path: hello, + apiOpts: []options.ApiOption{options.Api.Offline(true)}, + }, + { + name: "hashOnly", // test (non)fetchability + data: strFile(helloStr), + path: hello, + opts: []options.UnixfsAddOption{options.Unixfs.HashOnly(true)}, + }, + // multi file + { + name: "simpleDirNoWrap", + data: flatDir, + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + }, + { + name: "simpleDir", + data: flatDir, + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/Qmc3nGXm1HtUVCmnXLQHvWcNwfdZGpfg2SRm1CxLf7Q2Rm", + }, + { + name: "twoLevelDir", + data: twoLevelDir(), + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", + }, + // wrapped + { + name: "addWrapped", + path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE", + data: func() files.Node { + return files.NewBytesFile([]byte(helloStr)) + }, + wrap: "foo", + expect: wrapped("foo"), + }, + // hidden + { + name: "hiddenFilesAdded", + data: func() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + ".bar": files.NewBytesFile([]byte("hello2")), + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) + }, + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/QmPXLSBX382vJDLrGakcbrZDkU3grfkjMox7EgSC9KFbtQ", + }, + // NoCopy + { + name: "simpleNoCopy", + data: realFile, + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, + }, + { + name: "noCopyNoRaw", + data: realFile, + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true), options.Unixfs.RawLeaves(false)}, + err: "nocopy option requires '--raw-leaves' to be enabled as well", + }, + { + name: "noCopyNoPath", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, + err: helpers.ErrMissingFsRef.Error(), + }, + // Events / Progress + { + name: "simpleAddEvent", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + events: []coreiface.AddEvent{ + {Name: "bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", Path: p("bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa"), Size: strconv.Itoa(len(helloStr))}, + }, + opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)}, + }, + { + name: "silentAddEvent", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, + }, + opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, + }, + { + name: "dirAddEvents", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "abc/def", Path: p("QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk"), Size: "13"}, + {Name: "bar", Path: p("QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY"), Size: "14"}, + {Name: "foo", Path: p("QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ"), Size: "14"}, + {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, + }, + }, + { + name: "progress1M", + data: func() files.Node { + return files.NewReaderFile(bytes.NewReader(bytes.Repeat([]byte{0}, 1000000))) + }, + path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", + events: []coreiface.AddEvent{ + {Name: "", Bytes: 262144}, + {Name: "", Bytes: 524288}, + {Name: "", Bytes: 786432}, + {Name: "", Bytes: 1000000}, + {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Path: p("QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD"), Size: "1000256"}, + }, + wrap: "", + opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // recursive logic + + data := testCase.data() + if testCase.wrap != "" { + data = files.NewMapDirectory(map[string]files.Node{ + testCase.wrap: data, + }) + } + + // handle events if relevant to test case + + opts := testCase.opts + eventOut := make(chan interface{}) + var evtWg sync.WaitGroup + if len(testCase.events) > 0 { + opts = append(opts, options.Unixfs.Events(eventOut)) + evtWg.Add(1) + + go func() { + defer evtWg.Done() + expected := testCase.events + + for evt := range eventOut { + event, ok := evt.(*coreiface.AddEvent) + if !ok { + t.Error("unexpected event type") + continue + } + + if len(expected) < 1 { + t.Error("got more events than expected") + continue + } + + if expected[0].Size != event.Size { + t.Errorf("Event.Size didn't match, %s != %s", expected[0].Size, event.Size) + } + + if expected[0].Name != event.Name { + t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) + } + + if expected[0].Path != nil && event.Path != nil { + if expected[0].Path.Cid().String() != event.Path.Cid().String() { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) + } + } else if event.Path != expected[0].Path { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) + } + if expected[0].Bytes != event.Bytes { + t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes) + } + + expected = expected[1:] + } + + if len(expected) > 0 { + t.Errorf("%d event(s) didn't arrive", len(expected)) + } + }() + } + + tapi, err := api.WithOptions(testCase.apiOpts...) + if err != nil { + t.Fatal(err) + } + + // Add! + + p, err := tapi.Unixfs().Add(ctx, data, opts...) + close(eventOut) + evtWg.Wait() + if testCase.err != "" { + if err == nil { + t.Fatalf("expected an error: %s", testCase.err) + } + if err.Error() != testCase.err { + t.Fatalf("expected an error: '%s' != '%s'", err.Error(), testCase.err) + } + return + } + if err != nil { + t.Fatal(err) + } + + if p.String() != testCase.path { + t.Errorf("expected path %s, got: %s", testCase.path, p) + } + + // compare file structure with Unixfs().Get + + var cmpFile func(origName string, orig files.Node, gotName string, got files.Node) + cmpFile = func(origName string, orig files.Node, gotName string, got files.Node) { + _, origDir := orig.(files.Directory) + _, gotDir := got.(files.Directory) + + if origName != gotName { + t.Errorf("file name mismatch, orig='%s', got='%s'", origName, gotName) + } + + if origDir != gotDir { + t.Fatalf("file type mismatch on %s", origName) + } + + if !gotDir { + defer orig.Close() + defer got.Close() + + do, err := io.ReadAll(orig.(files.File)) + if err != nil { + t.Fatal(err) + } + + dg, err := io.ReadAll(got.(files.File)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(do, dg) { + t.Fatal("data not equal") + } + + return + } + + origIt := orig.(files.Directory).Entries() + gotIt := got.(files.Directory).Entries() + + for { + if origIt.Next() { + if !gotIt.Next() { + t.Fatal("gotIt out of entries before origIt") + } + } else { + if gotIt.Next() { + t.Fatal("origIt out of entries before gotIt") + } + break + } + + cmpFile(origIt.Name(), origIt.Node(), gotIt.Name(), gotIt.Node()) + } + if origIt.Err() != nil { + t.Fatal(origIt.Err()) + } + if gotIt.Err() != nil { + t.Fatal(gotIt.Err()) + } + } + + f, err := tapi.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + orig := testCase.data() + if testCase.expect != nil { + orig = testCase.expect(orig) + } + + cmpFile("", orig, "", f) + }) + } +} + +func (tp *TestSuite) TestAddPinned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.Pin(true)) + if err != nil { + t.Fatal(err) + } + + pins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + if len(pins) != 1 { + t.Fatalf("expected 1 pin, got %d", len(pins)) + } + + if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { + t.Fatalf("got unexpected pin: %s", pins[0].Path().String()) + } +} + +func (tp *TestSuite) TestAddHashOnly(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.HashOnly(true)) + if err != nil { + t.Fatal(err) + } + + if p.String() != hello { + t.Errorf("unxepected path: %s", p.String()) + } + + _, err = api.Block().Get(ctx, p) + if err == nil { + t.Fatal("expected an error") + } + if !ipld.IsNotFound(err) { + t.Errorf("unxepected error: %s", err.Error()) + } +} + +func (tp *TestSuite) TestGetEmptyFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, files.NewBytesFile([]byte{})) + if err != nil { + t.Fatal(err) + } + + emptyFilePath := path.New(emptyFile) + + r, err := api.Unixfs().Get(ctx, emptyFilePath) + if err != nil { + t.Fatal(err) + } + + buf := make([]byte, 1) // non-zero so that Read() actually tries to read + n, err := io.ReadFull(r.(files.File), buf) + if err != nil && err != io.EOF { + t.Error(err) + } + if !bytes.HasPrefix(buf, []byte{0x00}) { + t.Fatalf("expected empty data, got [%s] [read=%d]", buf, n) + } +} + +func (tp *TestSuite) TestGetDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + edir := unixfs.EmptyDirNode() + err = api.Dag().Add(ctx, edir) + if err != nil { + t.Fatal(err) + } + p := path.IpfsPath(edir.Cid()) + + emptyDir, err := api.Object().New(ctx, options.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + if p.String() != path.IpfsPath(emptyDir.Cid()).String() { + t.Fatalf("expected path %s, got: %s", emptyDir.Cid(), p.String()) + } + + r, err := api.Unixfs().Get(ctx, path.IpfsPath(emptyDir.Cid())) + if err != nil { + t.Fatal(err) + } + + if _, ok := r.(files.Directory); !ok { + t.Fatalf("expected a directory") + } +} + +func (tp *TestSuite) TestGetNonUnixfs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd := new(mdag.ProtoNode) + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Get(ctx, path.IpfsPath(nd.Cid())) + if !strings.Contains(err.Error(), "proto: required field") { + t.Fatalf("expected protobuf error, got: %s", err) + } +} + +func (tp *TestSuite) TestLs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + r := strings.NewReader("content-of-file") + p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ + "name-of-file": files.NewReaderFile(r), + "name-of-symlink": files.NewLinkFile("/foo/bar", nil), + })) + if err != nil { + t.Fatal(err) + } + + entries, err := api.Unixfs().Ls(ctx, p) + if err != nil { + t.Fatal(err) + } + + entry := <-entries + if entry.Err != nil { + t.Fatal(entry.Err) + } + if entry.Size != 15 { + t.Errorf("expected size = 15, got %d", entry.Size) + } + if entry.Name != "name-of-file" { + t.Errorf("expected name = name-of-file, got %s", entry.Name) + } + if entry.Type != coreiface.TFile { + t.Errorf("wrong type %s", entry.Type) + } + if entry.Cid.String() != "QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr" { + t.Errorf("expected cid = QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr, got %s", entry.Cid) + } + entry = <-entries + if entry.Err != nil { + t.Fatal(entry.Err) + } + if entry.Type != coreiface.TSymlink { + t.Errorf("wrong type %s", entry.Type) + } + if entry.Name != "name-of-symlink" { + t.Errorf("expected name = name-of-symlink, got %s", entry.Name) + } + if entry.Target != "/foo/bar" { + t.Errorf("expected symlink target to be /foo/bar, got %s", entry.Target) + } + + if l, ok := <-entries; ok { + t.Errorf("didn't expect a second link") + if l.Err != nil { + t.Error(l.Err) + } + } +} + +func (tp *TestSuite) TestEntriesExpired(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + r := strings.NewReader("content-of-file") + p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ + "name-of-file": files.NewReaderFile(r), + })) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithCancel(ctx) + + nd, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + cancel() + + it := files.ToDir(nd).Entries() + if it == nil { + t.Fatal("it was nil") + } + + if it.Next() { + t.Fatal("Next succeeded") + } + + if it.Err() != context.Canceled { + t.Fatalf("unexpected error %s", it.Err()) + } + + if it.Next() { + t.Fatal("Next succeeded") + } +} + +func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, files.NewSliceDirectory([]files.DirEntry{})) + if err != nil { + t.Fatal(err) + } + + emptyDir, err := api.Object().New(ctx, options.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + links, err := api.Unixfs().Ls(ctx, path.IpfsPath(emptyDir.Cid())) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Fatalf("expected 0 links, got %d", len(links)) + } +} + +// TODO(lgierth) this should test properly, with len(links) > 0 +func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := cbor.WrapObject(map[string]interface{}{"foo": "bar"}, math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + links, err := api.Unixfs().Ls(ctx, path.IpfsPath(nd.Cid())) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Fatalf("expected 0 links, got %d", len(links)) + } +} + +type closeTestF struct { + files.File + closed bool + + t *testing.T +} + +type closeTestD struct { + files.Directory + closed bool + + t *testing.T +} + +func (f *closeTestD) Close() error { + f.t.Helper() + if f.closed { + f.t.Fatal("already closed") + } + f.closed = true + return nil +} + +func (f *closeTestF) Close() error { + if f.closed { + f.t.Fatal("already closed") + } + f.closed = true + return nil +} + +func (tp *TestSuite) TestAddCloses(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + n4 := &closeTestF{files.NewBytesFile([]byte("foo")), false, t} + d3 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ + "sub": n4, + }), false, t} + n2 := &closeTestF{files.NewBytesFile([]byte("bar")), false, t} + n1 := &closeTestF{files.NewBytesFile([]byte("baz")), false, t} + d0 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ + "a": d3, + "b": n1, + "c": n2, + }), false, t} + + _, err = api.Unixfs().Add(ctx, d0) + if err != nil { + t.Fatal(err) + } + + for i, n := range []*closeTestF{n1, n2, n4} { + if !n.closed { + t.Errorf("file %d not closed!", i) + } + } + + for i, n := range []*closeTestD{d0, d3} { + if !n.closed { + t.Errorf("dir %d not closed!", i) + } + } +} + +func (tp *TestSuite) TestGetSeek(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + dataSize := int64(100000) + tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) + + p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f := files.ToFile(r) + if f == nil { + t.Fatal("not a file") + } + + orig := make([]byte, dataSize) + if _, err := io.ReadFull(f, orig); err != nil { + t.Fatal(err) + } + f.Close() + + origR := bytes.NewReader(orig) + + r, err = api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f = files.ToFile(r) + if f == nil { + t.Fatal("not a file") + } + + test := func(offset int64, whence int, read int, expect int64, shouldEof bool) { + t.Run(fmt.Sprintf("seek%d+%d-r%d-%d", whence, offset, read, expect), func(t *testing.T) { + n, err := f.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + origN, err := origR.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + + if n != origN { + t.Fatalf("offsets didn't match, expected %d, got %d", origN, n) + } + + buf := make([]byte, read) + origBuf := make([]byte, read) + origRead, err := origR.Read(origBuf) + if err != nil { + t.Fatalf("orig: %s", err) + } + r, err := io.ReadFull(f, buf) + switch { + case shouldEof && err != nil && err != io.ErrUnexpectedEOF: + fallthrough + case !shouldEof && err != nil: + t.Fatalf("f: %s", err) + case shouldEof: + _, err := f.Read([]byte{0}) + if err != io.EOF { + t.Fatal("expected EOF") + } + _, err = origR.Read([]byte{0}) + if err != io.EOF { + t.Fatal("expected EOF (orig)") + } + } + + if int64(r) != expect { + t.Fatal("read wrong amount of data") + } + if r != origRead { + t.Fatal("read different amount of data than bytes.Reader") + } + if !bytes.Equal(buf, origBuf) { + fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) + fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) + t.Fatal("data didn't match") + } + }) + } + + test(3, io.SeekCurrent, 10, 10, false) + test(3, io.SeekCurrent, 10, 10, false) + test(500, io.SeekCurrent, 10, 10, false) + test(350, io.SeekStart, 100, 100, false) + test(-123, io.SeekCurrent, 100, 100, false) + test(0, io.SeekStart, int(dataSize), dataSize, false) + test(dataSize-50, io.SeekStart, 100, 50, true) + test(-5, io.SeekEnd, 100, 5, true) +} + +func (tp *TestSuite) TestGetReadAt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + dataSize := int64(100000) + tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) + + p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f, ok := r.(interface { + files.File + io.ReaderAt + }) + if !ok { + t.Skip("ReaderAt not implemented") + } + + orig := make([]byte, dataSize) + if _, err := io.ReadFull(f, orig); err != nil { + t.Fatal(err) + } + f.Close() + + origR := bytes.NewReader(orig) + + if _, err := api.Unixfs().Get(ctx, p); err != nil { + t.Fatal(err) + } + + test := func(offset int64, read int, expect int64, shouldEof bool) { + t.Run(fmt.Sprintf("readat%d-r%d-%d", offset, read, expect), func(t *testing.T) { + origBuf := make([]byte, read) + origRead, err := origR.ReadAt(origBuf, offset) + if err != nil && err != io.EOF { + t.Fatalf("orig: %s", err) + } + buf := make([]byte, read) + r, err := f.ReadAt(buf, offset) + if shouldEof { + if err != io.EOF { + t.Fatal("expected EOF, got: ", err) + } + } else if err != nil { + t.Fatal("got: ", err) + } + + if int64(r) != expect { + t.Fatal("read wrong amount of data") + } + if r != origRead { + t.Fatal("read different amount of data than bytes.Reader") + } + if !bytes.Equal(buf, origBuf) { + fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) + fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) + t.Fatal("data didn't match") + } + }) + } + + test(3, 10, 10, false) + test(13, 10, 10, false) + test(513, 10, 10, false) + test(350, 100, 100, false) + test(0, int(dataSize), dataSize, false) + test(dataSize-50, 100, 50, true) +} diff --git a/core/coreiface/unixfs.go b/core/coreiface/unixfs.go new file mode 100644 index 000000000..606bc8e78 --- /dev/null +++ b/core/coreiface/unixfs.go @@ -0,0 +1,80 @@ +package iface + +import ( + "context" + + "github.com/ipfs/boxo/coreiface/options" + path "github.com/ipfs/boxo/coreiface/path" + + "github.com/ipfs/boxo/files" + "github.com/ipfs/go-cid" +) + +type AddEvent struct { + Name string + Path path.Resolved `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +// FileType is an enum of possible UnixFS file types. +type FileType int32 + +const ( + // TUnknown means the file type isn't known (e.g., it hasn't been + // resolved). + TUnknown FileType = iota + // TFile is a regular file. + TFile + // TDirectory is a directory. + TDirectory + // TSymlink is a symlink. + TSymlink +) + +func (t FileType) String() string { + switch t { + case TUnknown: + return "unknown" + case TFile: + return "file" + case TDirectory: + return "directory" + case TSymlink: + return "symlink" + default: + return "" + } +} + +// DirEntry is a directory entry returned by `Ls`. +type DirEntry struct { + Name string + Cid cid.Cid + + // Only filled when asked to resolve the directory entry. + Size uint64 // The size of the file in bytes (or the size of the symlink). + Type FileType // The type of the file. + Target string // The symlink target (if a symlink). + + Err error +} + +// UnixfsAPI is the basic interface to immutable files in IPFS +// NOTE: This API is heavily WIP, things are guaranteed to break frequently +type UnixfsAPI interface { + // Add imports the data from the reader into merkledag file + // + // TODO: a long useful comment on how to use this for many different scenarios + Add(context.Context, files.Node, ...options.UnixfsAddOption) (path.Resolved, error) + + // Get returns a read-only handle to a file tree referenced by a path + // + // Note that some implementations of this API may apply the specified context + // to operations performed on the returned file + Get(context.Context, path.Path) (files.Node, error) + + // Ls returns the list of links in a directory. Links aren't guaranteed to be + // returned in order + Ls(context.Context, path.Path, ...options.UnixfsLsOption) (<-chan DirEntry, error) +} diff --git a/core/coreiface/util.go b/core/coreiface/util.go new file mode 100644 index 000000000..6d58bf40d --- /dev/null +++ b/core/coreiface/util.go @@ -0,0 +1,20 @@ +package iface + +import ( + "context" + "io" +) + +type Reader interface { + ReadSeekCloser + Size() uint64 + CtxReadFull(context.Context, []byte) (int, error) +} + +// A ReadSeekCloser implements interfaces to read, copy, seek and close. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer + io.WriterTo +}