From 434871ba18662db366907e358654728a357f9c0e Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 9 Jun 2015 10:03:52 -0700 Subject: [PATCH] core/commands/unixfs: Add 'ipfs unixfs ls ...' This is similar to 'ipfs ls ...', but it: * Lists file sizes that match the content size: $ ipfs --encoding=json unixfs ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 { "Objects": [ { "Argument": "/ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4", "Links": [ { "Name": "busybox", "Hash": "QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a", "Size": 1947624, "Type": 2 } ] } ] } $ ipfs cat /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4/busybox | wc -c 1947624 'ipfs ls ...', on the other hand, is using the Merkle-descendant size, which also includes fanout links and the typing information unixfs objects store in their Data: $ ipfs --encoding=json ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 { "Objects": [ { "Hash": "/ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4", "Links": [ { "Name": "busybox", "Hash": "QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a", "Size": 1948128, "Type": 2 } ] } ] } * Has a simpler text output corresponding to POSIX ls [1]: $ ipfs unixfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02 bin dev etc proc run sys $ ipfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02 QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 1948183 bin/ QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4 dev/ QmUz1Z5jnQEjwr78fiMk5babwjJBDmhN5sx5HvPiTGGGjM 1207 etc/ QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4 proc/ QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4 run/ QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4 sys/ The minimal output allows us to start off with POSIX compliance and then add options (which may or may not be POSIX compatible) to adjust the output format as we get a better feel for what we need ([2] through [3]). [1]: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/ls.html [2]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41724727&page=5 [3]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41725146&page=5 License: MIT Signed-off-by: W. Trevor King --- core/commands/root.go | 3 + core/commands/unixfs/ls.go | 124 +++++++++++++++++++++++++++++++ core/commands/unixfs/unixfs.go | 21 ++++++ test/sharness/t0200-unixfs-ls.sh | 75 +++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 core/commands/unixfs/ls.go create mode 100644 core/commands/unixfs/unixfs.go create mode 100755 test/sharness/t0200-unixfs-ls.sh diff --git a/core/commands/root.go b/core/commands/root.go index 916b32b63..0c91208eb 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -5,6 +5,7 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs" evlog "github.com/ipfs/go-ipfs/thirdparty/eventlog" ) @@ -35,6 +36,7 @@ DATA STRUCTURE COMMANDS block Interact with raw blocks in the datastore object Interact with raw dag nodes + unixfs Interact with Unix filesystem objects ADVANCED COMMANDS @@ -102,6 +104,7 @@ var rootSubcommands = map[string]*cmds.Command{ "stats": StatsCmd, "swarm": SwarmCmd, "tour": tourCmd, + "unixfs": unixfs.UnixFSCmd, "update": UpdateCmd, "version": VersionCmd, "bitswap": BitswapCmd, diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go new file mode 100644 index 000000000..58ad2f811 --- /dev/null +++ b/core/commands/unixfs/ls.go @@ -0,0 +1,124 @@ +package unixfs + +import ( + "bytes" + "fmt" + "io" + "text/tabwriter" + "time" + + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + + cmds "github.com/ipfs/go-ipfs/commands" + core "github.com/ipfs/go-ipfs/core" + path "github.com/ipfs/go-ipfs/path" + unixfs "github.com/ipfs/go-ipfs/unixfs" + unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" +) + +type LsLink struct { + Name, Hash string + Size uint64 + Type unixfspb.Data_DataType +} + +type LsObject struct { + Argument string + Links []LsLink +} + +type LsOutput struct { + Objects []*LsObject +} + +var LsCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List directory contents for Unix-filesystem objects", + ShortDescription: ` +Retrieves the object named by and displays the +contents with the following format: + + + +For files, the child size is the total size of the file contents. For +directories, the child size is the IPFS link size. +`, + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to list links from").EnableStdin(), + }, + Run: func(req cmds.Request, res cmds.Response) { + node, err := req.Context().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + paths := req.Arguments() + + output := make([]*LsObject, len(paths)) + for i, fpath := range paths { + dagnode, err := core.Resolve(req.Context().Context, node, path.Path(fpath)) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + output[i] = &LsObject{ + Argument: fpath, + Links: make([]LsLink, len(dagnode.Links)), + } + for j, link := range dagnode.Links { + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) + defer cancel() + link.Node, err = link.GetNode(ctx, node.DAG) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + d, err := unixfs.FromBytes(link.Node.Data) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + lsLink := LsLink{ + Name: link.Name, + Hash: link.Hash.B58String(), + Type: d.GetType(), + } + if lsLink.Type == unixfspb.Data_File { + lsLink.Size = d.GetFilesize() + } else { + lsLink.Size = link.Size + } + output[i].Links[j] = lsLink + } + } + + res.SetOutput(&LsOutput{Objects: output}) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + + output := res.Output().(*LsOutput) + buf := new(bytes.Buffer) + w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) + for _, object := range output.Objects { + if len(output.Objects) > 1 { + fmt.Fprintf(w, "%s:\n", object.Argument) + } + for _, link := range object.Links { + fmt.Fprintf(w, "%s\n", link.Name) + } + if len(output.Objects) > 1 { + fmt.Fprintln(w) + } + } + w.Flush() + + return buf, nil + }, + }, + Type: LsOutput{}, +} diff --git a/core/commands/unixfs/unixfs.go b/core/commands/unixfs/unixfs.go new file mode 100644 index 000000000..de97c8a33 --- /dev/null +++ b/core/commands/unixfs/unixfs.go @@ -0,0 +1,21 @@ +package unixfs + +import cmds "github.com/ipfs/go-ipfs/commands" + +var UnixFSCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Interact with ipfs objects representing Unix filesystems", + ShortDescription: ` +'ipfs unixfs' provides a familar interface to filesystems represtented +by IPFS objects that hides IPFS-implementation details like layout +objects (e.g. fanout and chunking). +`, + Synopsis: ` +ipfs unixfs ls ... - List directory contents for ... +`, + }, + + Subcommands: map[string]*cmds.Command{ + "ls": LsCmd, + }, +} diff --git a/test/sharness/t0200-unixfs-ls.sh b/test/sharness/t0200-unixfs-ls.sh new file mode 100755 index 000000000..f5dc20d30 --- /dev/null +++ b/test/sharness/t0200-unixfs-ls.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test unixfs ls command" + +. lib/test-lib.sh + +test_init_ipfs + +test_ls_cmd() { + + test_expect_success "'ipfs add -r testData' succeeds" ' + mkdir -p testData testData/d1 testData/d2 && + echo "test" >testData/f1 && + echo "data" >testData/f2 && + echo "hello" >testData/d1/a && + random 128 42 >testData/d1/128 && + echo "world" >testData/d2/a && + random 1024 42 >testData/d2/1024 && + ipfs add -r testData >actual_add + ' + + test_expect_success "'ipfs add' output looks good" ' + cat <<-\EOF >expected_add && + added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128 + added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a + added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 + added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024 + added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a + added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 + added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 + added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 + added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData + EOF + test_cmp expected_add actual_add + ' + + test_expect_success "'ipfs unixfs ls ' succeeds" ' + ipfs unixfs ls QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls + ' + + test_expect_success "'ipfs unixfs ls ' output looks good" ' + cat <<-\EOF >expected_ls && + QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj: + d1 + d2 + f1 + f2 + + QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy: + 1024 + a + + QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: + 128 + a + + EOF + test_cmp expected_ls actual_ls + ' +} + + +# should work offline +test_ls_cmd + +# should work online +test_launch_ipfs_daemon +test_ls_cmd +test_kill_ipfs_daemon + +test_done