From 7203c43b60ef63e4e1e5dea26a300c888a7d7ea9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Jul 2017 11:48:59 -0700 Subject: [PATCH] plugin: create plugin API and loader, add ipld-git plugin License: MIT Signed-off-by: Jeromy --- Rules.mk | 3 ++ cmd/ipfs/main.go | 7 ++++ core/commands/dag/dag.go | 36 +++++++--------- core/coredag/dagtransl.go | 81 ++++++++++++++++++++++++++++++++++++ package.json | 6 +++ plugin/Rules.mk | 9 ++++ plugin/ipld.go | 16 +++++++ plugin/loader/.gitignore | 1 + plugin/loader/Rules.mk | 10 +++++ plugin/loader/initializer.go | 45 ++++++++++++++++++++ plugin/loader/load.go | 49 ++++++++++++++++++++++ plugin/loader/load_linux.go | 64 ++++++++++++++++++++++++++++ plugin/loader/preload.sh | 31 ++++++++++++++ plugin/loader/preload_list | 6 +++ plugin/plugin.go | 12 ++++++ plugin/plugins/.gitignore | 1 + plugin/plugins/Rules.mk | 14 +++++++ plugin/plugins/git/git.go | 63 ++++++++++++++++++++++++++++ 18 files changed, 432 insertions(+), 22 deletions(-) create mode 100644 core/coredag/dagtransl.go create mode 100644 plugin/Rules.mk create mode 100644 plugin/ipld.go create mode 100644 plugin/loader/.gitignore create mode 100644 plugin/loader/Rules.mk create mode 100644 plugin/loader/initializer.go create mode 100644 plugin/loader/load.go create mode 100644 plugin/loader/load_linux.go create mode 100755 plugin/loader/preload.sh create mode 100644 plugin/loader/preload_list create mode 100644 plugin/plugin.go create mode 100644 plugin/plugins/.gitignore create mode 100644 plugin/plugins/Rules.mk create mode 100644 plugin/plugins/git/git.go diff --git a/Rules.mk b/Rules.mk index 441e7ec4a..640e301b8 100644 --- a/Rules.mk +++ b/Rules.mk @@ -56,6 +56,9 @@ include $(dir)/Rules.mk dir := pin/internal/pb include $(dir)/Rules.mk +dir := plugin +include $(dir)/Rules.mk + # -------------------- # # universal rules # # -------------------- # diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index f555e0e12..dcb2df407 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "runtime/pprof" "strings" "sync" @@ -22,6 +23,7 @@ import ( cmdsHttp "github.com/ipfs/go-ipfs/commands/http" core "github.com/ipfs/go-ipfs/core" coreCmds "github.com/ipfs/go-ipfs/core/commands" + "github.com/ipfs/go-ipfs/plugin/loader" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" @@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd } else { log.Debug("executing command locally") + pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins") + if _, err := loader.LoadPlugins(pluginpath); err != nil { + return nil, err + } + err := req.SetRootContext(ctx) if err != nil { return nil, err diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 842ec3574..f61f3a1e5 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,6 +7,7 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + coredag "github.com/ipfs/go-ipfs/core/coredag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" @@ -76,34 +77,25 @@ into an object of the specified format. defer n.Blockstore.PinLock().Unlock() } + nds, err := coredag.ParseInputs(ienc, format, fi) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + var c *cid.Cid - switch ienc { - case "json": - nd, err := convertJsonToType(fi, format) + b := n.DAG.Batch() + for _, nd := range nds { + cid, err := b.Add(nd) if err != nil { res.SetError(err, cmds.ErrNormal) return } - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - case "raw": - nd, err := convertRawToType(fi, format) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - default: - res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal) + c = cid + } + if err := b.Commit(); err != nil { + res.SetError(err, cmds.ErrNormal) return } diff --git a/core/coredag/dagtransl.go b/core/coredag/dagtransl.go new file mode 100644 index 000000000..607d9af27 --- /dev/null +++ b/core/coredag/dagtransl.go @@ -0,0 +1,81 @@ +package coredag + +import ( + "fmt" + "io" + "io/ioutil" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" + ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor" +) + +type DagParser func(r io.Reader) ([]node.Node, error) + +type FormatParsers map[string]DagParser +type InputEncParsers map[string]FormatParsers + +var DefaultInputEncParsers = InputEncParsers{ + "json": DefaultJsonParsers, + "raw": DefaultRawParsers, +} + +var DefaultJsonParsers = FormatParsers{ + "cbor": CborJsonParser, + "dag-cbor": CborJsonParser, +} + +var DefaultRawParsers = FormatParsers{ + "cbor": CborRawParser, + "dag-cbor": CborRawParser, +} + +func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + return DefaultInputEncParsers.ParseInputs(ienc, format, r) +} + +func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) { + m, ok := iep[ienv] + if !ok { + m = make(FormatParsers) + iep[ienv] = m + } + + m[format] = f +} + +func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + pset, ok := iep[ienc] + if !ok { + return nil, fmt.Errorf("no input parser for %q", ienc) + } + + parser, ok := pset[format] + if !ok { + return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc) + } + + return parser(r) +} + +func CborJsonParser(r io.Reader) ([]node.Node, error) { + nd, err := ipldcbor.FromJson(r) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} + +func CborRawParser(r io.Reader) ([]node.Node, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + nd, err := ipldcbor.Decode(data) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} diff --git a/package.json b/package.json index bfc39d027..8f0e7a351 100644 --- a/package.json +++ b/package.json @@ -441,6 +441,12 @@ "hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1", "name": "go-libp2p-routing", "version": "2.2.17" + }, + { + "author": "whyrusleeping", + "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex", + "name": "go-ipld-git", + "version": "0.1.3" } ], "gxVersion": "0.10.0", diff --git a/plugin/Rules.mk b/plugin/Rules.mk new file mode 100644 index 000000000..1e26d2a3c --- /dev/null +++ b/plugin/Rules.mk @@ -0,0 +1,9 @@ +include mk/header.mk + +dir := $(d)/loader +include $(dir)/Rules.mk + +dir := $(d)/plugins +include $(dir)/Rules.mk + +include mk/footer.mk diff --git a/plugin/ipld.go b/plugin/ipld.go new file mode 100644 index 000000000..3dfdc0e04 --- /dev/null +++ b/plugin/ipld.go @@ -0,0 +1,16 @@ +package plugin + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +// PluginIPLD is an interface that can be implemented to add handlers for +// for different IPLD formats +type PluginIPLD interface { + Plugin + + RegisterBlockDecoders(dec node.BlockDecoder) error + RegisterInputEncParsers(iec coredag.InputEncParsers) error +} diff --git a/plugin/loader/.gitignore b/plugin/loader/.gitignore new file mode 100644 index 000000000..5f0355638 --- /dev/null +++ b/plugin/loader/.gitignore @@ -0,0 +1 @@ +preload.go diff --git a/plugin/loader/Rules.mk b/plugin/loader/Rules.mk new file mode 100644 index 000000000..50e4e743f --- /dev/null +++ b/plugin/loader/Rules.mk @@ -0,0 +1,10 @@ +include mk/header.mk + +$(d)/preload.go: d:=$(d) +$(d)/preload.go: $(d)/preload_list + $(d)/preload.sh > $@ + go fmt $@ >/dev/null + +DEPS_GO += $(d)/preload.go + +include mk/footer.mk diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go new file mode 100644 index 000000000..d8ee8aade --- /dev/null +++ b/plugin/loader/initializer.go @@ -0,0 +1,45 @@ +package loader + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + "github.com/ipfs/go-ipfs/plugin" + + format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +func initalize(plugins []plugin.Plugin) error { + for _, p := range plugins { + err := p.Init() + if err != nil { + return err + } + } + + return nil +} + +func run(plugins []plugin.Plugin) error { + for _, pl := range plugins { + err := runIPLDPlugin(pl) + if err != nil { + return err + } + } + return nil +} + +func runIPLDPlugin(pl plugin.Plugin) error { + ipldpl, ok := pl.(plugin.PluginIPLD) + if !ok { + return nil + } + + var err error + err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder) + if err != nil { + return err + } + + err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers) + return err +} diff --git a/plugin/loader/load.go b/plugin/loader/load.go new file mode 100644 index 000000000..d1489ccc6 --- /dev/null +++ b/plugin/loader/load.go @@ -0,0 +1,49 @@ +package loader + +import ( + "fmt" + + "github.com/ipfs/go-ipfs/plugin" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" +) + +var log = logging.Logger("plugin/loader") + +var loadPluginsFunc = func(string) ([]plugin.Plugin, error) { + return nil, nil +} + +// LoadPlugins loads and initalizes plugins. +func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { + plMap := make(map[string]plugin.Plugin) + for _, v := range preloadPlugins { + plMap[v.Name()] = v + } + + newPls, err := loadPluginsFunc(pluginDir) + if err != nil { + return nil, err + } + + for _, pl := range newPls { + if ppl, ok := plMap[pl.Name()]; ok { + // plugin is already preloaded + return nil, fmt.Errorf("plugin: %s, is duplicated in version: %s, while trying to load dynamically: %s", ppl.Name(), ppl.Version(), pl.Version()) + } + plMap[pl.Name()] = pl + } + + pls := make([]plugin.Plugin, 0, len(plMap)) + for _, v := range plMap { + pls = append(pls, v) + } + + err = initalize(pls) + if err != nil { + return nil, err + } + + err = run(pls) + return nil, err +} diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go new file mode 100644 index 000000000..511b42d53 --- /dev/null +++ b/plugin/loader/load_linux.go @@ -0,0 +1,64 @@ +package loader + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "plugin" + + iplugin "github.com/ipfs/go-ipfs/plugin" +) + +func init() { + loadPluginsFunc = linxuLoadFunc +} + +func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { + var plugins []iplugin.Plugin + + filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + log.Warningf("found directory inside plugins directory: %s", fi) + return nil + } + + if info.Mode().Perm()&0111 == 0 { + // file is not executable let's not load it + // this is to prevent loading plugins from for example non-executable + // mounts, some /tmp mounts are marked as such for security + log.Warningf("non-executable file in plugins directory: %s", fi) + return nil + } + + if newPlugins, err := loadPlugin(fi); err == nil { + plugins = append(plugins, newPlugins...) + } else { + return fmt.Errorf("loading plugin %s: %s", fi, err) + } + return nil + }) + + return plugins, nil +} + +func loadPlugin(fi string) ([]iplugin.Plugin, error) { + pl, err := plugin.Open(fi) + if err != nil { + return nil, err + } + pls, err := pl.Lookup("Plugins") + if err != nil { + return nil, err + } + + typePls, ok := pls.([]iplugin.Plugin) + if !ok { + return nil, errors.New("filed 'Plugins' didn't contain correct type") + } + + return typePls, nil +} diff --git a/plugin/loader/preload.sh b/plugin/loader/preload.sh new file mode 100755 index 000000000..dab8cfee4 --- /dev/null +++ b/plugin/loader/preload.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +to_preload() { + awk 'NF' "$DIR/preload_list" | sed '/^#/d' +} + +cat <