plugin: create plugin API and loader, add ipld-git plugin

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

Rules.mk

@@ -56,6 +56,9 @@ include $(dir)/Rules.mk
 dir := pin/internal/pb
 include $(dir)/Rules.mk
 
+dir := plugin
+include $(dir)/Rules.mk
+
 # -------------------- #
 #   universal rules    #
 # -------------------- #

cmd/ipfs/main.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime/pprof"
 	"strings"
 	"sync"
@@ -22,6 +23,7 @@ import (
 	cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
 	core "github.com/ipfs/go-ipfs/core"
 	coreCmds "github.com/ipfs/go-ipfs/core/commands"
+	"github.com/ipfs/go-ipfs/plugin/loader"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
@@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
 	} else {
 		log.Debug("executing command locally")
 
+		pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins")
+		if _, err := loader.LoadPlugins(pluginpath); err != nil {
+			return nil, err
+		}
+
 		err := req.SetRootContext(ctx)
 		if err != nil {
 			return nil, err

core/commands/dag/dag.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	coredag "github.com/ipfs/go-ipfs/core/coredag"
 	path "github.com/ipfs/go-ipfs/path"
 	pin "github.com/ipfs/go-ipfs/pin"
 
@@ -76,37 +77,28 @@ into an object of the specified format.
 			defer n.Blockstore.PinLock().Unlock()
 		}
 
+		nds, err := coredag.ParseInputs(ienc, format, fi)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
 		var c *cid.Cid
-		switch ienc {
-		case "json":
-			nd, err := convertJsonToType(fi, format)
+		b := n.DAG.Batch()
+		for _, nd := range nds {
+			cid, err := b.Add(nd)
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return
 			}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
+			c = cid
 		}
-		case "raw":
-			nd, err := convertRawToType(fi, format)
-			if err != nil {
+		if err := b.Commit(); err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
-		default:
-			res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
-			return
-		}
-
 		if dopin {
 			n.Pinning.PinWithMode(c, pin.Recursive)
 

core/coredag/dagtransl.go

@@ -0,0 +1,81 @@
+package coredag
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+	ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor"
+)
+
+type DagParser func(r io.Reader) ([]node.Node, error)
+
+type FormatParsers map[string]DagParser
+type InputEncParsers map[string]FormatParsers
+
+var DefaultInputEncParsers = InputEncParsers{
+	"json": DefaultJsonParsers,
+	"raw":  DefaultRawParsers,
+}
+
+var DefaultJsonParsers = FormatParsers{
+	"cbor":     CborJsonParser,
+	"dag-cbor": CborJsonParser,
+}
+
+var DefaultRawParsers = FormatParsers{
+	"cbor":     CborRawParser,
+	"dag-cbor": CborRawParser,
+}
+
+func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	return DefaultInputEncParsers.ParseInputs(ienc, format, r)
+}
+
+func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
+	m, ok := iep[ienv]
+	if !ok {
+		m = make(FormatParsers)
+		iep[ienv] = m
+	}
+
+	m[format] = f
+}
+
+func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	pset, ok := iep[ienc]
+	if !ok {
+		return nil, fmt.Errorf("no input parser for %q", ienc)
+	}
+
+	parser, ok := pset[format]
+	if !ok {
+		return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
+	}
+
+	return parser(r)
+}
+
+func CborJsonParser(r io.Reader) ([]node.Node, error) {
+	nd, err := ipldcbor.FromJson(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}
+
+func CborRawParser(r io.Reader) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd, err := ipldcbor.Decode(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}

package.json

@@ -441,6 +441,12 @@
       "hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1",
       "name": "go-libp2p-routing",
       "version": "2.2.17"
+    },
+    {
+      "author": "whyrusleeping",
+      "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex",
+      "name": "go-ipld-git",
+      "version": "0.1.3"
     }
   ],
   "gxVersion": "0.10.0",

plugin/Rules.mk

@@ -0,0 +1,9 @@
+include mk/header.mk
+
+dir := $(d)/loader
+include $(dir)/Rules.mk
+
+dir := $(d)/plugins
+include $(dir)/Rules.mk
+
+include mk/footer.mk

plugin/ipld.go

@@ -0,0 +1,16 @@
+package plugin
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+)
+
+// PluginIPLD is an interface that can be implemented to add handlers for
+// for different IPLD formats
+type PluginIPLD interface {
+	Plugin
+
+	RegisterBlockDecoders(dec node.BlockDecoder) error
+	RegisterInputEncParsers(iec coredag.InputEncParsers) error
+}

plugin/loader/.gitignore

@@ -0,0 +1 @@
+preload.go

plugin/loader/Rules.mk

@@ -0,0 +1,10 @@
+include mk/header.mk
+
+$(d)/preload.go: d:=$(d)
+$(d)/preload.go: $(d)/preload_list
+	$(d)/preload.sh > $@
+	go fmt $@ >/dev/null
+
+DEPS_GO += $(d)/preload.go
+	
+include mk/footer.mk

plugin/loader/initializer.go

@@ -0,0 +1,45 @@
+package loader
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+	"github.com/ipfs/go-ipfs/plugin"
+
+	format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+)
+
+func initalize(plugins []plugin.Plugin) error {
+	for _, p := range plugins {
+		err := p.Init()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func run(plugins []plugin.Plugin) error {
+	for _, pl := range plugins {
+		err := runIPLDPlugin(pl)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runIPLDPlugin(pl plugin.Plugin) error {
+	ipldpl, ok := pl.(plugin.PluginIPLD)
+	if !ok {
+		return nil
+	}
+
+	var err error
+	err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder)
+	if err != nil {
+		return err
+	}
+
+	err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers)
+	return err
+}

plugin/loader/load.go

@@ -0,0 +1,49 @@
+package loader
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-ipfs/plugin"
+
+	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+)
+
+var log = logging.Logger("plugin/loader")
+
+var loadPluginsFunc = func(string) ([]plugin.Plugin, error) {
+	return nil, nil
+}
+
+// LoadPlugins loads and initalizes plugins.
+func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) {
+	plMap := make(map[string]plugin.Plugin)
+	for _, v := range preloadPlugins {
+		plMap[v.Name()] = v
+	}
+
+	newPls, err := loadPluginsFunc(pluginDir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, pl := range newPls {
+		if ppl, ok := plMap[pl.Name()]; ok {
+			// plugin is already preloaded
+			return nil, fmt.Errorf("plugin: %s, is duplicated in version: %s, while trying to load dynamically: %s", ppl.Name(), ppl.Version(), pl.Version())
+		}
+		plMap[pl.Name()] = pl
+	}
+
+	pls := make([]plugin.Plugin, 0, len(plMap))
+	for _, v := range plMap {
+		pls = append(pls, v)
+	}
+
+	err = initalize(pls)
+	if err != nil {
+		return nil, err
+	}
+
+	err = run(pls)
+	return nil, err
+}

plugin/loader/load_linux.go

@@ -0,0 +1,64 @@
+package loader
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"plugin"
+
+	iplugin "github.com/ipfs/go-ipfs/plugin"
+)
+
+func init() {
+	loadPluginsFunc = linxuLoadFunc
+}
+
+func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) {
+	var plugins []iplugin.Plugin
+
+	filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			log.Warningf("found directory inside plugins directory: %s", fi)
+			return nil
+		}
+
+		if info.Mode().Perm()&0111 == 0 {
+			// file is not executable let's not load it
+			// this is to prevent loading plugins from for example non-executable
+			// mounts, some /tmp mounts are marked as such for security
+			log.Warningf("non-executable file in plugins directory: %s", fi)
+			return nil
+		}
+
+		if newPlugins, err := loadPlugin(fi); err == nil {
+			plugins = append(plugins, newPlugins...)
+		} else {
+			return fmt.Errorf("loading plugin %s: %s", fi, err)
+		}
+		return nil
+	})
+
+	return plugins, nil
+}
+
+func loadPlugin(fi string) ([]iplugin.Plugin, error) {
+	pl, err := plugin.Open(fi)
+	if err != nil {
+		return nil, err
+	}
+	pls, err := pl.Lookup("Plugins")
+	if err != nil {
+		return nil, err
+	}
+
+	typePls, ok := pls.([]iplugin.Plugin)
+	if !ok {
+		return nil, errors.New("filed 'Plugins' didn't contain correct type")
+	}
+
+	return typePls, nil
+}

plugin/loader/preload.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+to_preload() {
+	awk 'NF' "$DIR/preload_list" | sed '/^#/d'
+}
+
+cat <<EOL
+package loader
+
+import (
+	"github.com/ipfs/go-ipfs/plugin"
+EOL
+
+to_preload | while read -r name path num; do
+	echo "\tplugin$name \"$path\""
+done | sort -u
+
+cat <<EOL
+)
+
+var preloadPlugins = []plugin.Plugin{
+EOL
+
+to_preload | while read -r name path num; do
+	echo "\tplugin$name.Plugins[$num],"
+done
+
+
+echo "}"

plugin/loader/preload_list

@@ -0,0 +1,6 @@
+# this file contains plugins to be preloaded
+# empty lines or starting with '#' are ignored
+#
+# name             go-path                  number of the sub-plugin
+
+#ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0

plugin/plugin.go

@@ -0,0 +1,12 @@
+package plugin
+
+// Plugin is base interface for all kinds of go-ipfs plugins
+// It will be included in interfaces of different Plugins
+type Plugin interface {
+	// Name should return uniqe name of the plugin
+	Name() string
+	// Version returns current version of the plugin
+	Version() string
+	// Init is called once when the Plugin is being loaded
+	Init() error
+}

plugin/plugins/.gitignore

@@ -0,0 +1 @@
+*.so

plugin/plugins/Rules.mk

@@ -0,0 +1,14 @@
+include mk/header.mk
+
+$(d)_plugins:=$(d)/git
+$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
+
+$($(d)_plugins_so): $$(DEPS_GO) ALWAYS
+	go build -buildmode=plugin -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))"
+
+CLEAN += $($(d)_plugins_so)
+
+build_plugins: $($(d)_plugins_so)
+
+
+include mk/footer.mk

plugin/plugins/git/git.go

@@ -0,0 +1,63 @@
+package git
+
+import (
+	"compress/zlib"
+	"io"
+
+	"github.com/ipfs/go-ipfs/core/coredag"
+	"github.com/ipfs/go-ipfs/plugin"
+
+	"gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	"gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+	git "gx/ipfs/Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex/go-ipld-git"
+)
+
+var Plugins = []plugin.Plugin{
+	&GitPlugin{},
+}
+
+type GitPlugin struct{}
+
+var _ plugin.PluginIPLD = (*GitPlugin)(nil)
+
+func (*GitPlugin) Name() string {
+	return "ipld-git"
+}
+
+func (*GitPlugin) Version() string {
+	return "0.0.1"
+}
+
+func (*GitPlugin) Init() error {
+	return nil
+}
+
+func (*GitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error {
+	dec.Register(cid.GitRaw, git.DecodeBlock)
+	return nil
+}
+
+func (*GitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error {
+	iec.AddParser("raw", "git", parseRawGit)
+	iec.AddParser("zlib", "git", parseZlibGit)
+	return nil
+}
+
+func parseRawGit(r io.Reader) ([]format.Node, error) {
+	nd, err := git.ParseObject(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []format.Node{nd}, nil
+}
+
+func parseZlibGit(r io.Reader) ([]format.Node, error) {
+	rc, err := zlib.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rc.Close()
+	return parseRawGit(rc)
+}