From dfde39151f8c3645c2ed1dfa3217e9337b4cd5a9 Mon Sep 17 00:00:00 2001 From: Overbool Date: Sat, 27 Oct 2018 11:57:26 +0800 Subject: [PATCH 1/5] commands/filestore: use new cmds lib License: MIT Signed-off-by: Overbool --- core/commands/filestore.go | 95 ++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 51 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 9bff637cf..05622ebed 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -5,12 +5,9 @@ import ( "fmt" "io" - oldCmds "github.com/ipfs/go-ipfs/commands" - lgc "github.com/ipfs/go-ipfs/commands/legacy" - "github.com/ipfs/go-ipfs/core" + core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" - e "github.com/ipfs/go-ipfs/core/commands/e" - "github.com/ipfs/go-ipfs/filestore" + filestore "github.com/ipfs/go-ipfs/filestore" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds" @@ -23,8 +20,8 @@ var FileStoreCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ "ls": lsFileStore, - "verify": lgc.NewCommand(verifyFileStore), - "dups": lgc.NewCommand(dupsFileStore), + "verify": verifyFileStore, + "dups": dupsFileStore, }, } @@ -88,7 +85,7 @@ The output is: Type: filestore.ListRes{}, } -var verifyFileStore = &oldCmds.Command{ +var verifyFileStore = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Verify objects in filestore.", LongDescription: ` @@ -118,69 +115,54 @@ For ERROR entries the error will also be printed to stderr. Options: []cmdkit.Option{ cmdkit.BoolOption(fileOrderOptionName, "verify the objects based on the order of the backing file"), }, - Run: func(req oldCmds.Request, res oldCmds.Response) { - _, fs, err := getFilestore(req.InvocContext()) + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + _, fs, err := getFilestore(env) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } - args := req.Arguments() + args := req.Arguments if len(args) > 0 { - out := perKeyActionToChan(req.Context(), args, func(c cid.Cid) *filestore.ListRes { + out := perKeyActionToChan(req.Context, args, func(c cid.Cid) *filestore.ListRes { return filestore.Verify(fs, c) }) - res.SetOutput(out) + return res.Emit(out) } else { - fileOrder, _, _ := req.Option(fileOrderOptionName).Bool() + fileOrder, _ := req.Options[fileOrderOptionName].(bool) next, err := filestore.VerifyAll(fs, fileOrder) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } - out := listResToChan(req.Context(), next) - res.SetOutput(out) + out := listResToChan(req.Context, next) + return res.Emit(out) } }, - Marshalers: oldCmds.MarshalerMap{ - oldCmds.Text: func(res oldCmds.Response) (io.Reader, error) { - v, err := unwrapOutput(res.Output()) - if err != nil { - return nil, err + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error { + if out.Status == filestore.StatusOtherError { + return fmt.Errorf(out.ErrorMsg) } - - r, ok := v.(*filestore.ListRes) - if !ok { - return nil, e.TypeErr(r, v) - } - - if r.Status == filestore.StatusOtherError { - fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg) - } - fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong()) - return nil, nil - }, + fmt.Fprintf(w, "%s %s\n", out.Status.Format(), out.FormatLong()) + return nil + }), }, Type: filestore.ListRes{}, } -var dupsFileStore = &oldCmds.Command{ +var dupsFileStore = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "List blocks that are both in the filestore and standard block storage.", }, - Run: func(req oldCmds.Request, res oldCmds.Response) { - _, fs, err := getFilestore(req.InvocContext()) + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + _, fs, err := getFilestore(env) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } - ch, err := fs.FileManager().AllKeysChan(req.Context()) + ch, err := fs.FileManager().AllKeysChan(req.Context) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } out := make(chan interface{}, 128) - res.SetOutput((<-chan interface{})(out)) go func() { defer close(out) @@ -189,25 +171,36 @@ var dupsFileStore = &oldCmds.Command{ if err != nil { select { case out <- &RefWrapper{Err: err.Error()}: - case <-req.Context().Done(): + case <-req.Context.Done(): } return } if have { select { case out <- &RefWrapper{Ref: cid.String()}: - case <-req.Context().Done(): + case <-req.Context.Done(): return } } } }() + return res.Emit(out) }, - Marshalers: refsMarshallerMap, - Type: RefWrapper{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefWrapper) error { + if out.Err != "" { + return fmt.Errorf(out.Err) + } + + fmt.Fprintln(w, out.Ref) + + return nil + }), + }, + Type: RefWrapper{}, } -func getFilestore(env interface{}) (*core.IpfsNode, *filestore.Filestore, error) { +func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, error) { n, err := cmdenv.GetNode(env) if err != nil { return nil, nil, err From 013de54d9837e245bcc67ee8598fbcbd86cc1183 Mon Sep 17 00:00:00 2001 From: Overbool Date: Sat, 27 Oct 2018 12:53:04 +0800 Subject: [PATCH 2/5] commands/filestore: fix gofmt License: MIT Signed-off-by: Overbool --- core/commands/filestore.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 05622ebed..f89891c82 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -126,15 +126,15 @@ For ERROR entries the error will also be printed to stderr. return filestore.Verify(fs, c) }) return res.Emit(out) - } else { - fileOrder, _ := req.Options[fileOrderOptionName].(bool) - next, err := filestore.VerifyAll(fs, fileOrder) - if err != nil { - return err - } - out := listResToChan(req.Context, next) - return res.Emit(out) } + + fileOrder, _ := req.Options[fileOrderOptionName].(bool) + next, err := filestore.VerifyAll(fs, fileOrder) + if err != nil { + return err + } + out := listResToChan(req.Context, next) + return res.Emit(out) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error { From 2462bf913af34ccf05655ad1d691450f65922615 Mon Sep 17 00:00:00 2001 From: Overbool Date: Sun, 28 Oct 2018 11:30:01 +0800 Subject: [PATCH 3/5] commands/filestore: use res.Emit directly License: MIT Signed-off-by: Overbool --- core/commands/filestore.go | 128 +++++++++++++++---------------------- 1 file changed, 52 insertions(+), 76 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index f89891c82..e071fb4c7 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -1,7 +1,6 @@ package commands import ( - "context" "fmt" "io" @@ -56,11 +55,7 @@ The output is: } args := req.Arguments if len(args) > 0 { - out := perKeyActionToChan(req.Context, args, func(c cid.Cid) *filestore.ListRes { - return filestore.List(fs, c) - }) - - return res.Emit(out) + return listByArgs(res, fs, args) } fileOrder, _ := req.Options[fileOrderOptionName].(bool) @@ -69,8 +64,17 @@ The output is: return err } - out := listResToChan(req.Context, next) - return res.Emit(out) + for { + r := next() + if r == nil { + break + } + if err := res.Emit(r); err != nil { + return err + } + } + + return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError { @@ -122,10 +126,7 @@ For ERROR entries the error will also be printed to stderr. } args := req.Arguments if len(args) > 0 { - out := perKeyActionToChan(req.Context, args, func(c cid.Cid) *filestore.ListRes { - return filestore.Verify(fs, c) - }) - return res.Emit(out) + return listByArgs(res, fs, args) } fileOrder, _ := req.Options[fileOrderOptionName].(bool) @@ -133,8 +134,18 @@ For ERROR entries the error will also be printed to stderr. if err != nil { return err } - out := listResToChan(req.Context, next) - return res.Emit(out) + + for { + r := next() + if r == nil { + break + } + if err := res.Emit(r); err != nil { + return err + } + } + + return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error { @@ -162,29 +173,19 @@ var dupsFileStore = &cmds.Command{ return err } - out := make(chan interface{}, 128) - - go func() { - defer close(out) - for cid := range ch { - have, err := fs.MainBlockstore().Has(cid) - if err != nil { - select { - case out <- &RefWrapper{Err: err.Error()}: - case <-req.Context.Done(): - } - return - } - if have { - select { - case out <- &RefWrapper{Ref: cid.String()}: - case <-req.Context.Done(): - return - } + for cid := range ch { + have, err := fs.MainBlockstore().Has(cid) + if err != nil { + return res.Emit(&RefWrapper{Err: err.Error()}) + } + if have { + if err := res.Emit(&RefWrapper{Ref: cid.String()}); err != nil { + return err } } - }() - return res.Emit(out) + } + + return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefWrapper) error { @@ -212,49 +213,24 @@ func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, e return n, fs, err } -func listResToChan(ctx context.Context, next func() *filestore.ListRes) <-chan interface{} { - out := make(chan interface{}, 128) - go func() { - defer close(out) - for { - r := next() - if r == nil { - return +func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error { + for _, arg := range args { + c, err := cid.Decode(arg) + if err != nil { + ret := &filestore.ListRes{ + Status: filestore.StatusOtherError, + ErrorMsg: fmt.Sprintf("%s: %v", arg, err), } - select { - case out <- r: - case <-ctx.Done(): - return + if err := res.Emit(ret); err != nil { + return err } + continue } - }() - return out -} - -func perKeyActionToChan(ctx context.Context, args []string, action func(cid.Cid) *filestore.ListRes) <-chan interface{} { - out := make(chan interface{}, 128) - go func() { - defer close(out) - for _, arg := range args { - c, err := cid.Decode(arg) - if err != nil { - select { - case out <- &filestore.ListRes{ - Status: filestore.StatusOtherError, - ErrorMsg: fmt.Sprintf("%s: %v", arg, err), - }: - case <-ctx.Done(): - } - - continue - } - r := action(c) - select { - case out <- r: - case <-ctx.Done(): - return - } + r := filestore.Verify(fs, c) + if err := res.Emit(r); err != nil { + return err } - }() - return out + } + + return nil } From c43658621d57aeb492097b5c3ea4b1f1b82a1331 Mon Sep 17 00:00:00 2001 From: Overbool Date: Tue, 30 Oct 2018 14:26:01 +0800 Subject: [PATCH 4/5] commands/filestore: print StatusOtherError error License: MIT Signed-off-by: Overbool --- core/commands/filestore.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index e071fb4c7..fd83ee943 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "io" + "os" core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" @@ -150,7 +151,7 @@ For ERROR entries the error will also be printed to stderr. Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error { if out.Status == filestore.StatusOtherError { - return fmt.Errorf(out.ErrorMsg) + fmt.Fprintf(os.Stderr, "%s\n", out.ErrorMsg) } fmt.Fprintf(w, "%s %s\n", out.Status.Format(), out.FormatLong()) return nil From b4b6fc6906dd1279c1d09bef7383c7af3628194e Mon Sep 17 00:00:00 2001 From: Overbool Date: Sat, 3 Nov 2018 22:56:42 +0800 Subject: [PATCH 5/5] cmds/filestore: use PostRun in verify License: MIT Signed-off-by: Overbool --- core/commands/filestore.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index fd83ee943..70c02a3a4 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -7,6 +7,7 @@ import ( core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + e "github.com/ipfs/go-ipfs/core/commands/e" filestore "github.com/ipfs/go-ipfs/filestore" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" @@ -148,14 +149,28 @@ For ERROR entries the error will also be printed to stderr. return nil }, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error { - if out.Status == filestore.StatusOtherError { - fmt.Fprintf(os.Stderr, "%s\n", out.ErrorMsg) + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + for { + v, err := res.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + list, ok := v.(*filestore.ListRes) + if !ok { + return e.TypeErr(list, v) + } + + if list.Status == filestore.StatusOtherError { + fmt.Fprintf(os.Stderr, "%s\n", list.ErrorMsg) + } + fmt.Fprintf(os.Stdout, "%s %s\n", list.Status.Format(), list.FormatLong()) } - fmt.Fprintf(w, "%s %s\n", out.Status.Format(), out.FormatLong()) - return nil - }), + }, }, Type: filestore.ListRes{}, }