From 7203c43b60ef63e4e1e5dea26a300c888a7d7ea9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Jul 2017 11:48:59 -0700 Subject: [PATCH 01/23] plugin: create plugin API and loader, add ipld-git plugin License: MIT Signed-off-by: Jeromy --- Rules.mk | 3 ++ cmd/ipfs/main.go | 7 ++++ core/commands/dag/dag.go | 36 +++++++--------- core/coredag/dagtransl.go | 81 ++++++++++++++++++++++++++++++++++++ package.json | 6 +++ plugin/Rules.mk | 9 ++++ plugin/ipld.go | 16 +++++++ plugin/loader/.gitignore | 1 + plugin/loader/Rules.mk | 10 +++++ plugin/loader/initializer.go | 45 ++++++++++++++++++++ plugin/loader/load.go | 49 ++++++++++++++++++++++ plugin/loader/load_linux.go | 64 ++++++++++++++++++++++++++++ plugin/loader/preload.sh | 31 ++++++++++++++ plugin/loader/preload_list | 6 +++ plugin/plugin.go | 12 ++++++ plugin/plugins/.gitignore | 1 + plugin/plugins/Rules.mk | 14 +++++++ plugin/plugins/git/git.go | 63 ++++++++++++++++++++++++++++ 18 files changed, 432 insertions(+), 22 deletions(-) create mode 100644 core/coredag/dagtransl.go create mode 100644 plugin/Rules.mk create mode 100644 plugin/ipld.go create mode 100644 plugin/loader/.gitignore create mode 100644 plugin/loader/Rules.mk create mode 100644 plugin/loader/initializer.go create mode 100644 plugin/loader/load.go create mode 100644 plugin/loader/load_linux.go create mode 100755 plugin/loader/preload.sh create mode 100644 plugin/loader/preload_list create mode 100644 plugin/plugin.go create mode 100644 plugin/plugins/.gitignore create mode 100644 plugin/plugins/Rules.mk create mode 100644 plugin/plugins/git/git.go diff --git a/Rules.mk b/Rules.mk index 441e7ec4a..640e301b8 100644 --- a/Rules.mk +++ b/Rules.mk @@ -56,6 +56,9 @@ include $(dir)/Rules.mk dir := pin/internal/pb include $(dir)/Rules.mk +dir := plugin +include $(dir)/Rules.mk + # -------------------- # # universal rules # # -------------------- # diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index f555e0e12..dcb2df407 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "runtime/pprof" "strings" "sync" @@ -22,6 +23,7 @@ import ( cmdsHttp "github.com/ipfs/go-ipfs/commands/http" core "github.com/ipfs/go-ipfs/core" coreCmds "github.com/ipfs/go-ipfs/core/commands" + "github.com/ipfs/go-ipfs/plugin/loader" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" @@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd } else { log.Debug("executing command locally") + pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins") + if _, err := loader.LoadPlugins(pluginpath); err != nil { + return nil, err + } + err := req.SetRootContext(ctx) if err != nil { return nil, err diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 842ec3574..f61f3a1e5 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,6 +7,7 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + coredag "github.com/ipfs/go-ipfs/core/coredag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" @@ -76,34 +77,25 @@ into an object of the specified format. defer n.Blockstore.PinLock().Unlock() } + nds, err := coredag.ParseInputs(ienc, format, fi) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + var c *cid.Cid - switch ienc { - case "json": - nd, err := convertJsonToType(fi, format) + b := n.DAG.Batch() + for _, nd := range nds { + cid, err := b.Add(nd) if err != nil { res.SetError(err, cmds.ErrNormal) return } - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - case "raw": - nd, err := convertRawToType(fi, format) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - default: - res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal) + c = cid + } + if err := b.Commit(); err != nil { + res.SetError(err, cmds.ErrNormal) return } diff --git a/core/coredag/dagtransl.go b/core/coredag/dagtransl.go new file mode 100644 index 000000000..607d9af27 --- /dev/null +++ b/core/coredag/dagtransl.go @@ -0,0 +1,81 @@ +package coredag + +import ( + "fmt" + "io" + "io/ioutil" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" + ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor" +) + +type DagParser func(r io.Reader) ([]node.Node, error) + +type FormatParsers map[string]DagParser +type InputEncParsers map[string]FormatParsers + +var DefaultInputEncParsers = InputEncParsers{ + "json": DefaultJsonParsers, + "raw": DefaultRawParsers, +} + +var DefaultJsonParsers = FormatParsers{ + "cbor": CborJsonParser, + "dag-cbor": CborJsonParser, +} + +var DefaultRawParsers = FormatParsers{ + "cbor": CborRawParser, + "dag-cbor": CborRawParser, +} + +func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + return DefaultInputEncParsers.ParseInputs(ienc, format, r) +} + +func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) { + m, ok := iep[ienv] + if !ok { + m = make(FormatParsers) + iep[ienv] = m + } + + m[format] = f +} + +func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + pset, ok := iep[ienc] + if !ok { + return nil, fmt.Errorf("no input parser for %q", ienc) + } + + parser, ok := pset[format] + if !ok { + return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc) + } + + return parser(r) +} + +func CborJsonParser(r io.Reader) ([]node.Node, error) { + nd, err := ipldcbor.FromJson(r) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} + +func CborRawParser(r io.Reader) ([]node.Node, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + nd, err := ipldcbor.Decode(data) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} diff --git a/package.json b/package.json index bfc39d027..8f0e7a351 100644 --- a/package.json +++ b/package.json @@ -441,6 +441,12 @@ "hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1", "name": "go-libp2p-routing", "version": "2.2.17" + }, + { + "author": "whyrusleeping", + "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex", + "name": "go-ipld-git", + "version": "0.1.3" } ], "gxVersion": "0.10.0", diff --git a/plugin/Rules.mk b/plugin/Rules.mk new file mode 100644 index 000000000..1e26d2a3c --- /dev/null +++ b/plugin/Rules.mk @@ -0,0 +1,9 @@ +include mk/header.mk + +dir := $(d)/loader +include $(dir)/Rules.mk + +dir := $(d)/plugins +include $(dir)/Rules.mk + +include mk/footer.mk diff --git a/plugin/ipld.go b/plugin/ipld.go new file mode 100644 index 000000000..3dfdc0e04 --- /dev/null +++ b/plugin/ipld.go @@ -0,0 +1,16 @@ +package plugin + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +// PluginIPLD is an interface that can be implemented to add handlers for +// for different IPLD formats +type PluginIPLD interface { + Plugin + + RegisterBlockDecoders(dec node.BlockDecoder) error + RegisterInputEncParsers(iec coredag.InputEncParsers) error +} diff --git a/plugin/loader/.gitignore b/plugin/loader/.gitignore new file mode 100644 index 000000000..5f0355638 --- /dev/null +++ b/plugin/loader/.gitignore @@ -0,0 +1 @@ +preload.go diff --git a/plugin/loader/Rules.mk b/plugin/loader/Rules.mk new file mode 100644 index 000000000..50e4e743f --- /dev/null +++ b/plugin/loader/Rules.mk @@ -0,0 +1,10 @@ +include mk/header.mk + +$(d)/preload.go: d:=$(d) +$(d)/preload.go: $(d)/preload_list + $(d)/preload.sh > $@ + go fmt $@ >/dev/null + +DEPS_GO += $(d)/preload.go + +include mk/footer.mk diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go new file mode 100644 index 000000000..d8ee8aade --- /dev/null +++ b/plugin/loader/initializer.go @@ -0,0 +1,45 @@ +package loader + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + "github.com/ipfs/go-ipfs/plugin" + + format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +func initalize(plugins []plugin.Plugin) error { + for _, p := range plugins { + err := p.Init() + if err != nil { + return err + } + } + + return nil +} + +func run(plugins []plugin.Plugin) error { + for _, pl := range plugins { + err := runIPLDPlugin(pl) + if err != nil { + return err + } + } + return nil +} + +func runIPLDPlugin(pl plugin.Plugin) error { + ipldpl, ok := pl.(plugin.PluginIPLD) + if !ok { + return nil + } + + var err error + err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder) + if err != nil { + return err + } + + err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers) + return err +} diff --git a/plugin/loader/load.go b/plugin/loader/load.go new file mode 100644 index 000000000..d1489ccc6 --- /dev/null +++ b/plugin/loader/load.go @@ -0,0 +1,49 @@ +package loader + +import ( + "fmt" + + "github.com/ipfs/go-ipfs/plugin" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" +) + +var log = logging.Logger("plugin/loader") + +var loadPluginsFunc = func(string) ([]plugin.Plugin, error) { + return nil, nil +} + +// LoadPlugins loads and initalizes plugins. +func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { + plMap := make(map[string]plugin.Plugin) + for _, v := range preloadPlugins { + plMap[v.Name()] = v + } + + newPls, err := loadPluginsFunc(pluginDir) + if err != nil { + return nil, err + } + + for _, pl := range newPls { + if ppl, ok := plMap[pl.Name()]; ok { + // plugin is already preloaded + return nil, fmt.Errorf("plugin: %s, is duplicated in version: %s, while trying to load dynamically: %s", ppl.Name(), ppl.Version(), pl.Version()) + } + plMap[pl.Name()] = pl + } + + pls := make([]plugin.Plugin, 0, len(plMap)) + for _, v := range plMap { + pls = append(pls, v) + } + + err = initalize(pls) + if err != nil { + return nil, err + } + + err = run(pls) + return nil, err +} diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go new file mode 100644 index 000000000..511b42d53 --- /dev/null +++ b/plugin/loader/load_linux.go @@ -0,0 +1,64 @@ +package loader + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "plugin" + + iplugin "github.com/ipfs/go-ipfs/plugin" +) + +func init() { + loadPluginsFunc = linxuLoadFunc +} + +func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { + var plugins []iplugin.Plugin + + filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + log.Warningf("found directory inside plugins directory: %s", fi) + return nil + } + + if info.Mode().Perm()&0111 == 0 { + // file is not executable let's not load it + // this is to prevent loading plugins from for example non-executable + // mounts, some /tmp mounts are marked as such for security + log.Warningf("non-executable file in plugins directory: %s", fi) + return nil + } + + if newPlugins, err := loadPlugin(fi); err == nil { + plugins = append(plugins, newPlugins...) + } else { + return fmt.Errorf("loading plugin %s: %s", fi, err) + } + return nil + }) + + return plugins, nil +} + +func loadPlugin(fi string) ([]iplugin.Plugin, error) { + pl, err := plugin.Open(fi) + if err != nil { + return nil, err + } + pls, err := pl.Lookup("Plugins") + if err != nil { + return nil, err + } + + typePls, ok := pls.([]iplugin.Plugin) + if !ok { + return nil, errors.New("filed 'Plugins' didn't contain correct type") + } + + return typePls, nil +} diff --git a/plugin/loader/preload.sh b/plugin/loader/preload.sh new file mode 100755 index 000000000..dab8cfee4 --- /dev/null +++ b/plugin/loader/preload.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +to_preload() { + awk 'NF' "$DIR/preload_list" | sed '/^#/d' +} + +cat < Date: Wed, 12 Jul 2017 16:53:46 +0200 Subject: [PATCH 02/23] test: add TEST_NO_PLUGIN and PLUGIN prereq License: MIT Signed-off-by: Jakub Sztandera --- test/sharness/lib/test-lib.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index ac608c049..599bf0b32 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -40,12 +40,13 @@ SHARNESS_LIB="lib/sharness/sharness.sh" # Please put go-ipfs specific shell functions below +TEST_OS="$(uname -s | tr '[a-z]' '[A-Z]')" + # grab + output options test "$TEST_NO_FUSE" != 1 && test_set_prereq FUSE test "$TEST_EXPENSIVE" = 1 && test_set_prereq EXPENSIVE test "$TEST_NO_DOCKER" != 1 && type docker >/dev/null 2>&1 && test_set_prereq DOCKER - -TEST_OS=$(uname -s | tr [a-z] [A-Z]) +test "$TEST_NO_PLUGIN" != 1 && test "$TEST_OS" = "LINUX" && test_set_prereq PLUGIN # Set a prereq as error messages are often different on Windows/Cygwin expr "$TEST_OS" : "CYGWIN_NT" >/dev/null || test_set_prereq STD_ERR_MSG From d6f280ecec4d148f6c42163f842d79692c6abcdd Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 20:11:36 +0200 Subject: [PATCH 03/23] make: fix megacheck being run on help print License: MIT Signed-off-by: Jakub Sztandera --- Rules.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rules.mk b/Rules.mk index 640e301b8..24b00231f 100644 --- a/Rules.mk +++ b/Rules.mk @@ -145,7 +145,7 @@ help: @echo ' test_go_short' @echo ' test_go_expensive' @echo ' test_go_race' - @echo ' test_go_megacheck' - Run the `megacheck` vetting tool + @echo ' test_go_megacheck - Run the `megacheck` vetting tool' @echo ' test_sharness_short' @echo ' test_sharness_expensive' @echo ' test_sharness_race' From d2cc708650a2555fffea77cc3c446427f9e8da57 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 20:48:21 +0200 Subject: [PATCH 04/23] make(sharness): add plugins as part of sharness build License: MIT Signed-off-by: Jakub Sztandera --- Rules.mk | 6 ++++-- mk/util.mk | 1 + test/sharness/.gitignore | 1 + test/sharness/Rules.mk | 15 ++++++++++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Rules.mk b/Rules.mk index 24b00231f..ca9d1bf25 100644 --- a/Rules.mk +++ b/Rules.mk @@ -27,6 +27,10 @@ export IPFS_REUSEPORT=false dir := bin include $(dir)/Rules.mk +# tests need access to rules from plugin +dir := plugin +include $(dir)/Rules.mk + dir := test include $(dir)/Rules.mk @@ -56,8 +60,6 @@ include $(dir)/Rules.mk dir := pin/internal/pb include $(dir)/Rules.mk -dir := plugin -include $(dir)/Rules.mk # -------------------- # # universal rules # diff --git a/mk/util.mk b/mk/util.mk index 05e2dbe75..67b94f952 100644 --- a/mk/util.mk +++ b/mk/util.mk @@ -1,4 +1,5 @@ # util functions +OS ?= $(shell sh -c 'uname -s 2>/dev/null || echo not') ifeq ($(OS),Windows_NT) WINDOWS :=1 ?exe :=.exe # windows compat diff --git a/test/sharness/.gitignore b/test/sharness/.gitignore index 5e59048ac..06ceccc75 100644 --- a/test/sharness/.gitignore +++ b/test/sharness/.gitignore @@ -1,3 +1,4 @@ lib/sharness/ test-results/ trash directory.*.sh/ +plugins diff --git a/test/sharness/Rules.mk b/test/sharness/Rules.mk index 916f376c1..188cb50cb 100644 --- a/test/sharness/Rules.mk +++ b/test/sharness/Rules.mk @@ -1,6 +1,5 @@ include mk/header.mk - SHARNESS_$(d) = $(d)/lib/sharness/sharness.sh T_$(d) = $(sort $(wildcard $(d)/t[0-9][0-9][0-9][0-9]-*.sh)) @@ -12,6 +11,20 @@ DEPS_$(d) += cmd/ipfs/ipfs DEPS_$(d) += $(d)/clean-test-results DEPS_$(d) += $(SHARNESS_$(d)) +ifeq ($(OS),Linux) +PLUGINS_DIR_$(d) := $(d)/plugins/ +ORGIN_PLUGINS_$(d) := $(plugin/plugins_plugins_so) +PLUGINS_$(d) := $(addprefix $(PLUGINS_DIR_$(d)),$(notdir $(ORGIN_PLUGINS_$(d)))) + +$(PLUGINS_$(d)): $(ORGIN_PLUGINS_$(d)) + @mkdir -p $(@D) + cp -f plugin/plugins/$(@F) $@ + +ifneq ($(TEST_NO_PLUGIN),1) +DEPS_$(d) += $(PLUGINS_$(d)) +endif +endif + export MAKE_SKIP_PATH=1 $(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency From 2c71cf0fd7221f9976176395d40956d0088b8c57 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 21:02:55 +0200 Subject: [PATCH 05/23] test: add git plugin test stub License: MIT Signed-off-by: Jakub Sztandera --- test/sharness/t0280-plugin-git.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 test/sharness/t0280-plugin-git.sh diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh new file mode 100755 index 000000000..606a7efe4 --- /dev/null +++ b/test/sharness/t0280-plugin-git.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# +# Copyright (c) 2016 Jakub Sztandera +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test git plugin" + +. lib/test-lib.sh + +# if in travis CI, dont test mount (no fuse) +if ! test_have_prereq PLUGIN; then + skip_all='skipping git plugin tests, plugins not available' + + test_done +fi + +test_init_ipfs + +test_expect_success "copy plugin" ' + mkdir -p "$IPFS_PATH/plugins" && + cp ../plugins/git.so "$IPFS_PATH/plugins/" +' + +# test here + +test_done From 04b26fe08983428ae0afa3900275e93d6aa43eb8 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 22:47:01 +0200 Subject: [PATCH 06/23] make: fix buildmode=plugin and pkgdir problems License: MIT Signed-off-by: Jakub Sztandera --- plugin/plugins/Rules.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugins/Rules.mk b/plugin/plugins/Rules.mk index 29015e6c7..381696be0 100644 --- a/plugin/plugins/Rules.mk +++ b/plugin/plugins/Rules.mk @@ -4,7 +4,7 @@ $(d)_plugins:=$(d)/git $(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins)) $($(d)_plugins_so): $$(DEPS_GO) ALWAYS - go build -buildmode=plugin -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))" + go build -buildmode=plugin -i -pkgdir "$$GOPATH/pkg/linux_amd64_dynlink" $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))" CLEAN += $($(d)_plugins_so) From cfad4608279cc6d588589fa910b1b433e3b45670 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 23:03:15 +0200 Subject: [PATCH 07/23] git: make gitPlugin struct private License: MIT Signed-off-by: Jakub Sztandera --- plugin/plugins/git/git.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 61bb1eb69..5a0763301 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -13,31 +13,31 @@ import ( ) var Plugins = []plugin.Plugin{ - &GitPlugin{}, + &gitPlugin{}, } -type GitPlugin struct{} +type gitPlugin struct{} -var _ plugin.PluginIPLD = (*GitPlugin)(nil) +var _ plugin.PluginIPLD = (*gitPlugin)(nil) -func (*GitPlugin) Name() string { +func (*gitPlugin) Name() string { return "ipld-git" } -func (*GitPlugin) Version() string { +func (*gitPlugin) Version() string { return "0.0.1" } -func (*GitPlugin) Init() error { +func (*gitPlugin) Init() error { return nil } -func (*GitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error { +func (*gitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error { dec.Register(cid.GitRaw, git.DecodeBlock) return nil } -func (*GitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error { +func (*gitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error { iec.AddParser("raw", "git", parseRawGit) iec.AddParser("zlib", "git", parseZlibGit) return nil From d6657edf5900c7464f502ea06d834c7240150ae6 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 23:08:41 +0200 Subject: [PATCH 08/23] make codeclimate a bit more happy License: MIT Signed-off-by: Jakub Sztandera --- core/coredag/dagtransl.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/core/coredag/dagtransl.go b/core/coredag/dagtransl.go index 607d9af27..5a39e4656 100644 --- a/core/coredag/dagtransl.go +++ b/core/coredag/dagtransl.go @@ -9,30 +9,38 @@ import ( ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor" ) +// DagParser is function used for parsing stream into Node type DagParser func(r io.Reader) ([]node.Node, error) +// FormatParsers is used for mapping format descriptors to DagParsers type FormatParsers map[string]DagParser + +// InputEncParsers is used for mapping input encodings to FormatParsers type InputEncParsers map[string]FormatParsers +// DefaultInputEncParsers is InputEncParser that is used everywhere var DefaultInputEncParsers = InputEncParsers{ - "json": DefaultJsonParsers, - "raw": DefaultRawParsers, + "json": defaultJsonParsers, + "raw": defaultRawParsers, } -var DefaultJsonParsers = FormatParsers{ - "cbor": CborJsonParser, - "dag-cbor": CborJsonParser, +var defaultJsonParsers = FormatParsers{ + "cbor": cborJsonParser, + "dag-cbor": cborJsonParser, } -var DefaultRawParsers = FormatParsers{ - "cbor": CborRawParser, - "dag-cbor": CborRawParser, +var defaultRawParsers = FormatParsers{ + "cbor": cborRawParser, + "dag-cbor": cborRawParser, } +// ParseInputs uses DefaultInputEncParsers to parse io.Reader described by +// input encoding and format to an instance of ipld Node func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { return DefaultInputEncParsers.ParseInputs(ienc, format, r) } +// AddParser adds DagParser under give input encoding and format func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) { m, ok := iep[ienv] if !ok { @@ -43,6 +51,8 @@ func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) { m[format] = f } +// ParseInputs parses io.Reader described by input encoding and format to +// an instance of ipld Node func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { pset, ok := iep[ienc] if !ok { @@ -57,7 +67,7 @@ func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node return parser(r) } -func CborJsonParser(r io.Reader) ([]node.Node, error) { +func cborJsonParser(r io.Reader) ([]node.Node, error) { nd, err := ipldcbor.FromJson(r) if err != nil { return nil, err @@ -66,7 +76,7 @@ func CborJsonParser(r io.Reader) ([]node.Node, error) { return []node.Node{nd}, nil } -func CborRawParser(r io.Reader) ([]node.Node, error) { +func cborRawParser(r io.Reader) ([]node.Node, error) { data, err := ioutil.ReadAll(r) if err != nil { return nil, err From f947931af97811260cbaaa516ff09ae4c30d5229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Jul 2017 23:37:36 +0200 Subject: [PATCH 09/23] git: add git plugin tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- .../sharness/t0280-plugin-git-data/git.tar.gz | Bin 0 -> 4260 bytes test/sharness/t0280-plugin-git.sh | 32 +++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 test/sharness/t0280-plugin-git-data/git.tar.gz diff --git a/test/sharness/t0280-plugin-git-data/git.tar.gz b/test/sharness/t0280-plugin-git-data/git.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..ff5298b7785a3a3149519087a87de4fbf532c0d5 GIT binary patch literal 4260 zcmZXXbzIY5`^E=2K;i310Rcg}M~KK5kS;+=8tE44&Y>bqNu|4#E@`RJC?QfJ(hej> zj==_O`~2|tJpVoCpYzvwo%?m2>w4e!;fSXMmFy{6fXMdqLyZ|Ve+@&6m>I8&ak{Au z!GDv^U9IIeV?{JzFk_+y_cQ3+c2Qrt-I9Y^DY+^QBhwO{bcw?tr>|pz2N$Q-y@QIP z-|w{&-rg<|&y*cdz|x;c*sH9{X%mR;YkmGzRfSgiW{A@ti?ry~H_GU41C>ld1rnMs+?1_C+{xO3sM88jsua;_q^b2f+DPq^@Ytl8Yuo}eR@}Bt zygv%opd_>sMZ)x81>l-uM%;krB`NXuh2{e0>w^{OHO_OJL;8_qhBQu0pc>&zOUZc(Q-C+_Y%?-r_q8RHt_}oJT5F^C=QX>$^Rw6JD^>1`)zWTUs;bTD*&17vO$>b;vX~V3>;40eiCr3@?$KUzr8Huj{ML) zfMseFUsz78r*OIOPy*!%Rm2QQYEaTCuZP?^g4-8~92*!(K#fOItDgd@Pi2X(DgczS zI-b8KvOFqZj>o!9x;fiYcwxG`WLtd*&m?`NMvOaNlW82|DyyvWzKZVgJ0}4y3%lzt zNzpDQ)I}4olZUBVIKS4`N^V`5d z)1IH`vhZ6Q<8hds^(BJxN^!6hfklLrRGgH~sffp2&G{M}PW__Dy-MgSH5BeA{i|Xc zkR_huAP|x(H;C^8NmV}q=gm!8CNJGkyojKi-G=PDPPwu2@b78uL9xHBk}NU~#5N(r zX@Kk14a0|Ae~-;@jB3%Pe5sC+x#%f6RwBs)P2l>d@&>sRwX`ZVSWON9$#i@nl6e;h zGgt>|T+p!7)N=lOF``gc6y|K|schr?g#mE%Wa9GQE}-qS^PSlubZhcJ>g;t$|Un{jZhBii3(zTle}>X7?}S^C3&{P9Axm z`?Gv{b6<%c&85n4P*-bW^3H|r)sx1hm^r1t=9;8azm{`M3-W{Wn{!i}UgL~6Fs&}t zPDjCv-3iH8Q5ZlFU=-6J$AvPZ5+YO#zw2h7n8O@iIk#3V--mDsKH9y z@JN&l);l>Fg;8(wRV{q{@%60 zuF38ARQpG$AUP2Q-BE8lOTfYmA6Ed(U6&&8-n7#mB;DXuaT@e^s}fyNS;a+`;mkXS zV=kIi!*iHE1i)7Xs-{iU>FcF!0h+E)cms;8R>%I*{`)&SZ^!;7M@|VM&R`N z(7Y6RWuxIoP31%*Q@gmc9$HwPDuzoD3fpQ*V$~_QbMrA?6A8wjw#6PcW5AcGh7tfT zstQoT!UFqsVgJ~vJ20BKQ1W%TFuY=-onlsJls4?`;`7Xly_V4UrQ1_FzNX0@J&AXNWm$AIP7T=Ga-@kIl zX6A_O(Q}Y0`YlHeHt8JVQ)UK6$+7L9sx=qV+j4_v_o%CTu;?a?k|1=D0ca5ewrH-Z zgp6bOZ;$vX4d?VaRtPYo4aifFa5%UY4`G|6Z@KMhybnyGyz#ByPi#AmUgBV1Ns_bP z*&o>VJ;Abn#x9?t0&kdsSI=`q{0JD1%->$T1^J{*t2pAxH>=KY#GfTGWd2mChC8~v zP{fM1(AO_NBs{9L@Fv7F&$s*qs*x>ZC*n`NDD}e3FWXkk13~C;0CA?{x1xytUs{zR z$eljGMQHEEs!EPY8r&k86z(eXAvJWmeiu2Qe1#E7;-qO6@ak?Nu&t6dF`Yby0`47C zbuVm05QYkrS0lv{M4Cg)D_Pl@w!FoF3tp_SVatlb(ojCT4Q>o>9Sre=ZAE4-7|Oib z?c**r9k+;Cj@k~Rj<`hA&$l0v4SvGjzw#j;x8^kuS2#AN?+RDY9>4 z*5c=99w+PPdyQ`>mM!b#7r*(2*hd&%Z<49?S={`yE&=1fqEZSWJl*uCeQ+XoDdMF1 zTNB?V*8XpZ2ppDxGXFS6vn%!}azDWyPA(Ri8P33)h*sx#8lGUg5f)eGDwlWUb(!huDx{lr z9xz#tc;Nc&6`x*T>Vh5BHoC8mPJJT3{8iwUafg3%DC-X*eIJ;{BDP;;6HDB>E`)~Z zC_+e>Ij%8(Xn0wqLQWFH$l@?4ENuwvgNYD{31I3q(;jeOxwy0MYrl@{Og_g2d}uhS zp6Aw~kUL!Y(Xh1>GtXKZ)gXgQ%)itz(ms|U7Bb=Ty}$R~t@d8!TkX@n>osA}d5i3Z z3+1{(Y}q1Pk7B?IYNqT?gVZ?OJ=1H)*T^QJk|^hGLe!sx?Zjjg&0X$ttEA5{Xnd!f zHMel21ikO2`66zfLTm4_foHN3Bu?625Sgqiegc7LV4etl`sr@(w}EgRzCHtcs+(^1 zIz&Fg$h^M7Y7beRs*<_S^6Do4@~JrsGjjTwrDErHtllyGaMVD={q<*=L6C0c>5?zS z)k`A}4}Py&doc!g$8AU=Jk5_PHWCNe`C{bRWW?)3JdS>3zw(>xk@M5ns6aL}|6K3v zKKC79e>!~lGw-CweynmDnw04oaX7x#Uoc56*s7o^8W~8$UyTaSuNocbDz`7Qi79Rx z|In!2j&mxw%SMt{B$*Ix{F1kq`({P7acN=|7Pc^YiI}}IP6-k6a_%8$ZG)viBFb$l z7T3J6in09hImy*QC2CLCr^IK5l(qcqF`c|jmDo5AtZZ1w_NzVA%-t_Q1Gh9yX1m^J zaYi2J7BD@Xxdu(#!6(}}6aNdEUXq$G8k1Z15^0?i;dK0!LJT}C4MN^Ny$-Hea__&Km75`1GEWBPvzQ1QD>Rfv3#m}w8*IT=x(kt8RxkL=p z7P%+K%ED$?^W5feWD=jAjz1var;BQ9^_-N`d3#`Acvt27Qm*W1$p`2oy^q$q+ooOo z%i_V#^KT&R4v2i)=7Zc8*0Bim8RP2-!oy1&1H&iNavMb-p(}-;nR5=|G2j{o|2NN* zqw@UBiXr1;(a`$SdgW`38hi#$;yjI-jX6+=0SeU#k8NH@clR#7d#lYmAMk=WKE8VO)V`&OlwfRA-q|Q2vHvepsuhWZ%X-&N7-eJ%%`#6{jES_ zo>l{hw_wwPb1kP5h!)<2aTQ!3#PGs^Yq2QOX*+BtRAZ*KjbAl9v#`EH^P*iJ8!77D z|9*~&Eq3LSw=DuQ5{gpu;)1(6;k=?}S zSt|g)RVLss|DW0V8Jh-Wqd8aC4QBQGmOWgfFF&A_OwCP5sf8Qo+)8&(VVgL}1TY&X zkbBEQFe%Pbwj27oRuGPV-fC^9G#+zIACKx`N>9*_l@sSpemL=z)eudVmH!?)yxp%{ z8+R&;Yd^WTVBNDhvk9PT|0m0YF{esxwc&1Ir5XuYmdbV`3+lRMPILn zYbPa_3W1a6vVr@tW;YLlmd`ZQ2AW+u$yay8Wne$(t?I?t>ntv~@UJBl0IS7|!auOx zE4EM%2q-bA^$U=eUKe68kEXoK^@kDJO+t4^Nc#QNHycAhWN?kfjy;0b11_t8OJ->( zF&szsEvla9X5q2&E_9$PNz{EKc(ZX}Wv)%1%D0Z9t6-?c7VpqznaO@SK@^j-BoB(0o2P$n vm7n`a@3~xl1YxxFS0I0&RwL#}km1H_m0|;d|J}}?9O>(esu+U4fI$BTJ#@sa literal 0 HcmV?d00001 diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index 606a7efe4..4e2eaffea 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -22,6 +22,36 @@ test_expect_success "copy plugin" ' cp ../plugins/git.so "$IPFS_PATH/plugins/" ' -# test here +# from https://github.com/ipfs/go-ipld-git/blob/master/make-test-repo.sh +test_expect_success "prepare test data" ' + tar xzf ../t0280-plugin-git-data/git.tar.gz +' + +test_dag_git() { + test_expect_success "add objects via dag put" ' + find objects -type f -exec ipfs dag put --format=git --input-enc=zlib {} \; -exec echo \; > hashes + ' + + test_expect_success "successfully get added objects" ' + cat hashes | xargs -i ipfs dag get -- {} > /dev/null + ' + + test_expect_success "path traversals work" ' + echo \"YmxvYiA3ACcsLnB5Zgo=\" > file1 && + ipfs dag get z8mWaJh5RLq16Zwgtd8gZxd63P4hgwNNx/object/parents/0/tree/dir2/hash/f3/hash > out1 + ' + + test_expect_success "outputs look correct" ' + test_cmp file1 out1 + ' +} + +# should work offline +#test_dag_git + +# should work online +test_launch_ipfs_daemon +test_dag_git +test_kill_ipfs_daemon test_done From 42b0ba345f959c953d29ff1317a297ee4747dc59 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 12 Jul 2017 23:39:23 +0200 Subject: [PATCH 10/23] fixup plugin loading they have to be a main package License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/load_linux.go | 8 +++++--- plugin/plugins/Rules.mk | 1 + plugin/plugins/git/git.go | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go index 511b42d53..d74c91bf6 100644 --- a/plugin/loader/load_linux.go +++ b/plugin/loader/load_linux.go @@ -17,12 +17,14 @@ func init() { func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { var plugins []iplugin.Plugin - filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { + err := filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { - log.Warningf("found directory inside plugins directory: %s", fi) + if fi != pluginDir { + log.Warningf("found directory inside plugins directory: %s", fi) + } return nil } @@ -42,7 +44,7 @@ func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { return nil }) - return plugins, nil + return plugins, err } func loadPlugin(fi string) ([]iplugin.Plugin, error) { diff --git a/plugin/plugins/Rules.mk b/plugin/plugins/Rules.mk index 381696be0..570f0ba33 100644 --- a/plugin/plugins/Rules.mk +++ b/plugin/plugins/Rules.mk @@ -5,6 +5,7 @@ $(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins)) $($(d)_plugins_so): $$(DEPS_GO) ALWAYS go build -buildmode=plugin -i -pkgdir "$$GOPATH/pkg/linux_amd64_dynlink" $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))" + chmod +x "$@" CLEAN += $($(d)_plugins_so) diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 5a0763301..f7cab2a6a 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -1,4 +1,4 @@ -package git +package main import ( "compress/zlib" From b0679728f0d4c45e8270fc49cad60ee52ae05f15 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 08:49:11 +0200 Subject: [PATCH 11/23] make codeclimate more happy License: MIT Signed-off-by: Jakub Sztandera --- core/coredag/dagtransl.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/coredag/dagtransl.go b/core/coredag/dagtransl.go index 5a39e4656..5e0f5d808 100644 --- a/core/coredag/dagtransl.go +++ b/core/coredag/dagtransl.go @@ -20,13 +20,13 @@ type InputEncParsers map[string]FormatParsers // DefaultInputEncParsers is InputEncParser that is used everywhere var DefaultInputEncParsers = InputEncParsers{ - "json": defaultJsonParsers, + "json": defaultJSONParsers, "raw": defaultRawParsers, } -var defaultJsonParsers = FormatParsers{ - "cbor": cborJsonParser, - "dag-cbor": cborJsonParser, +var defaultJSONParsers = FormatParsers{ + "cbor": cborJSONParser, + "dag-cbor": cborJSONParser, } var defaultRawParsers = FormatParsers{ @@ -67,7 +67,7 @@ func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node return parser(r) } -func cborJsonParser(r io.Reader) ([]node.Node, error) { +func cborJSONParser(r io.Reader) ([]node.Node, error) { nd, err := ipldcbor.FromJson(r) if err != nil { return nil, err From c569b0be8ae5bcd85d915f8922cd7c4a489aabdf Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 21:14:24 +0200 Subject: [PATCH 12/23] plugin: make the plugin preload work again License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/preload.sh | 4 ++-- plugin/loader/preload_list | 2 +- plugin/plugins/.gitignore | 1 + plugin/plugins/Rules.mk | 9 ++++++++- plugin/plugins/gen_main.sh | 18 ++++++++++++++++++ plugin/plugins/git/git.go | 2 +- 6 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 plugin/plugins/gen_main.sh diff --git a/plugin/loader/preload.sh b/plugin/loader/preload.sh index dab8cfee4..93ee3e43b 100755 --- a/plugin/loader/preload.sh +++ b/plugin/loader/preload.sh @@ -14,7 +14,7 @@ import ( EOL to_preload | while read -r name path num; do - echo "\tplugin$name \"$path\"" + echo "plugin$name \"$path\"" done | sort -u cat < "$main_pkg/main.go" < Date: Thu, 13 Jul 2017 21:57:40 +0200 Subject: [PATCH 13/23] plugin: fix error when plugins dir does not exist License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/load.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugin/loader/load.go b/plugin/loader/load.go index d1489ccc6..2555277b0 100644 --- a/plugin/loader/load.go +++ b/plugin/loader/load.go @@ -2,6 +2,7 @@ package loader import ( "fmt" + "os" "github.com/ipfs/go-ipfs/plugin" @@ -21,7 +22,7 @@ func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { plMap[v.Name()] = v } - newPls, err := loadPluginsFunc(pluginDir) + newPls, err := loadDynamicPlugins(pluginDir) if err != nil { return nil, err } @@ -29,7 +30,10 @@ func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { for _, pl := range newPls { if ppl, ok := plMap[pl.Name()]; ok { // plugin is already preloaded - return nil, fmt.Errorf("plugin: %s, is duplicated in version: %s, while trying to load dynamically: %s", ppl.Name(), ppl.Version(), pl.Version()) + return nil, fmt.Errorf( + "plugin: %s, is duplicated in version: %s, "+ + "while trying to load dynamically: %s", + ppl.Name(), ppl.Version(), pl.Version()) } plMap[pl.Name()] = pl } @@ -47,3 +51,15 @@ func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { err = run(pls) return nil, err } + +func loadDynamicPlugins(pluginDir string) ([]plugin.Plugin, error) { + _, err := os.Stat(pluginDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + return loadPluginsFunc(pluginDir) +} From 314d2c3113c77606a3db687bda59900973ac08c4 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 21:59:01 +0200 Subject: [PATCH 14/23] misc: make codecov really happy License: MIT Signed-off-by: Jakub Sztandera --- plugin/plugins/git/git.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 5a0763301..5ec5d6f16 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -12,6 +12,7 @@ import ( git "gx/ipfs/Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex/go-ipld-git" ) +// Plugins is exported list of plugins that will be loaded var Plugins = []plugin.Plugin{ &gitPlugin{}, } From be5ee0b280eb64f389fbbd437d23ca9a93daa111 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 22:03:33 +0200 Subject: [PATCH 15/23] Disable ipldgit preload License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/preload_list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index 2905dfe1c..8c62c673b 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -3,4 +3,4 @@ # # name go-path number of the sub-plugin -ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0 +#ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0 From b135a1fa86cd20769fd1f1957a7eee212a900deb Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 22:05:31 +0200 Subject: [PATCH 16/23] misc: add go fmt to main packages of plugins License: MIT Signed-off-by: Jakub Sztandera --- plugin/plugins/Rules.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/plugins/Rules.mk b/plugin/plugins/Rules.mk index e571263dd..b5c9d43a2 100644 --- a/plugin/plugins/Rules.mk +++ b/plugin/plugins/Rules.mk @@ -8,6 +8,7 @@ $(d)_plugins_main:=$(addsuffix /main/main.go,$($(d)_plugins)) $($(d)_plugins_main): d:=$(d) $($(d)_plugins_main): $(d)/gen_main.sh "$(dir $@).." "$(call go-pkg-name,$(dir $@)/..)" + go fmt $@ >/dev/null $($(d)_plugins_so): %.so : %/main/main.go $($(d)_plugins_so): $$(DEPS_GO) ALWAYS From 6da6e7756e6e2ea8a6698eba0214dfcac8492074 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Jul 2017 23:02:13 +0200 Subject: [PATCH 17/23] plugin: fix plugin loading License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/load_linux.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go index d74c91bf6..98f0e5ec4 100644 --- a/plugin/loader/load_linux.go +++ b/plugin/loader/load_linux.go @@ -56,11 +56,12 @@ func loadPlugin(fi string) ([]iplugin.Plugin, error) { if err != nil { return nil, err } + log.Errorf("plugins: %T", pls) - typePls, ok := pls.([]iplugin.Plugin) + typePls, ok := pls.(*[]iplugin.Plugin) if !ok { return nil, errors.New("filed 'Plugins' didn't contain correct type") } - return typePls, nil + return *typePls, nil } From fb618a28e7328681ea2069aa9dfa45ffd29e30ef Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 14 Jul 2017 10:05:53 +0200 Subject: [PATCH 18/23] plugin: fix typos License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/initializer.go | 2 +- plugin/loader/load.go | 4 ++-- plugin/loader/load_linux.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go index d8ee8aade..2613738be 100644 --- a/plugin/loader/initializer.go +++ b/plugin/loader/initializer.go @@ -7,7 +7,7 @@ import ( format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" ) -func initalize(plugins []plugin.Plugin) error { +func initialize(plugins []plugin.Plugin) error { for _, p := range plugins { err := p.Init() if err != nil { diff --git a/plugin/loader/load.go b/plugin/loader/load.go index 2555277b0..8edd53675 100644 --- a/plugin/loader/load.go +++ b/plugin/loader/load.go @@ -15,7 +15,7 @@ var loadPluginsFunc = func(string) ([]plugin.Plugin, error) { return nil, nil } -// LoadPlugins loads and initalizes plugins. +// LoadPlugins loads and initializes plugins. func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { plMap := make(map[string]plugin.Plugin) for _, v := range preloadPlugins { @@ -43,7 +43,7 @@ func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { pls = append(pls, v) } - err = initalize(pls) + err = initialize(pls) if err != nil { return nil, err } diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go index 98f0e5ec4..4d1a18cf8 100644 --- a/plugin/loader/load_linux.go +++ b/plugin/loader/load_linux.go @@ -11,10 +11,10 @@ import ( ) func init() { - loadPluginsFunc = linxuLoadFunc + loadPluginsFunc = linuxLoadFunc } -func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { +func linuxLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { var plugins []iplugin.Plugin err := filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { From 36cbfa416dbb801b73510fd46e1f0bc030ace411 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 14 Jul 2017 11:45:53 +0200 Subject: [PATCH 19/23] coverage: add dependency on DEPS_GO for coverage License: MIT Signed-off-by: Jakub Sztandera --- coverage/Rules.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage/Rules.mk b/coverage/Rules.mk index bf433621c..1e797dfb9 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -1,6 +1,6 @@ include mk/header.mk -$(d)/coverage_deps: +$(d)/coverage_deps: $$(DEPS_GO) rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover ifneq ($(IPFS_SKIP_COVER_BINS),1) From 2df61862ac6d0210e0a1476d64b8029718c4b1ab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 14 Jul 2017 12:18:56 +0200 Subject: [PATCH 20/23] coverage: disable plugin tests during sharness coverage collection License: MIT Signed-off-by: Jakub Sztandera --- coverage/Rules.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/coverage/Rules.mk b/coverage/Rules.mk index 1e797dfb9..873342c40 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -41,6 +41,7 @@ endif export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ +$(D)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1 $(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ From b5ea00e6b4871ea3fa4f3a4fcc58c1f72a928f07 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 14 Jul 2017 13:23:14 +0200 Subject: [PATCH 21/23] sharness: add debug output for TEST_NO_PLUGIN License: MIT Signed-off-by: Jakub Sztandera --- test/sharness/lib/test-lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 599bf0b32..01aac0932 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -54,6 +54,7 @@ expr "$TEST_OS" : "CYGWIN_NT" >/dev/null || test_set_prereq STD_ERR_MSG if test "$TEST_VERBOSE" = 1; then echo '# TEST_VERBOSE='"$TEST_VERBOSE" echo '# TEST_NO_FUSE='"$TEST_NO_FUSE" + echo '# TEST_NO_PLUGIN='"$TEST_NO_PLUGIN" echo '# TEST_EXPENSIVE='"$TEST_EXPENSIVE" echo '# TEST_OS='"$TEST_OS" fi From 3e8e03939f011dd19e173e75c24b2a1d0e3580be Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 14 Jul 2017 16:46:22 +0200 Subject: [PATCH 22/23] fix typo License: MIT Signed-off-by: Jakub Sztandera --- coverage/Rules.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage/Rules.mk b/coverage/Rules.mk index 873342c40..f137b18c9 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -41,7 +41,7 @@ endif export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ -$(D)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1 +$(d)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1 $(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ From c0ee7d68fd76c0691b2086e5727be832d802f8f3 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 16 Jul 2017 10:26:34 +0200 Subject: [PATCH 23/23] misc: small code style edit License: MIT Signed-off-by: Jakub Sztandera --- plugin/loader/initializer.go | 6 ++---- test/sharness/t0280-plugin-git.sh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go index 2613738be..7ad627eef 100644 --- a/plugin/loader/initializer.go +++ b/plugin/loader/initializer.go @@ -34,12 +34,10 @@ func runIPLDPlugin(pl plugin.Plugin) error { return nil } - var err error - err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder) + err := ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder) if err != nil { return err } - err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers) - return err + return ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers) } diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index 4e2eaffea..f09efed66 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2016 Jakub Sztandera +# Copyright (c) 2017 Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. #