diff --git a/Makefile b/Makefile index a746944f5..2ac4d0949 100644 --- a/Makefile +++ b/Makefile @@ -1,195 +1,10 @@ -# Minimum version numbers for software required to build IPFS -IPFS_MIN_GO_VERSION = 1.7 -IPFS_MIN_GX_VERSION = 0.6 -IPFS_MIN_GX_GO_VERSION = 1.1 +# General tools -GOTAGS = -GOTAGS += "" # we have to have always at least one tag, empty tag works well +SHELL=PATH=$(PATH) /bin/sh -GOFLAGS = -GOTFLAGS = +PROTOC = protoc --gogo_out=. --proto_path=.:/usr/local/opt/protobuf/include:$(dir $@) $< -export IPFS_REUSEPORT=false -export GOFLAGS -export GOTFLAGS +# enable second expansion +.SECONDEXPANSION: -GOFLAGS += -tags $(call join-with,$(comma),$(GOTAGS)) - -ifeq ($(TEST_NO_FUSE),1) - GOTAGS += nofuse -endif - -ifeq ($(OS),Windows_NT) - GOPATH_DELIMITER = ; -else - GOPATH_DELIMITER = : -endif - -dist_root=/ipfs/QmNZL8wNsvAGdVYr8uGeUE9aGfHjFpHegAWywQFEdSaJbp -gx_bin=bin/gx-v0.9.0 -gx-go_bin=bin/gx-go-v1.3.0 - - -# util functions - -space = -space += -comma =, -join-with = $(subst $(space),$1,$(strip $2)) -# use things in our bin before any other system binaries -export PATH := bin:$(PATH) -export IPFS_API ?= v04x.ipfs.io - -all: help - -godep: - go get github.com/tools/godep - -go_check: - @bin/check_go_version $(IPFS_MIN_GO_VERSION) - -bin/gx-v%: - @echo "installing gx $(@:bin/gx-%=%)" - @bin/dist_get ${dist_root} gx $@ $(@:bin/gx-%=%) - rm -f bin/gx - ln -s $(@:bin/%=%) bin/gx - -bin/gx-go-v%: - @echo "installing gx-go $(@:bin/gx-go-%=%)" - @bin/dist_get ${dist_root} gx-go $@ $(@:bin/gx-go-%=%) - rm -f bin/gx-go - ln -s $(@:bin/%=%) bin/gx-go - -gx_check: ${gx_bin} ${gx-go_bin} - -path_check: - @bin/check_go_path $(realpath $(shell pwd)) $(realpath $(addsuffix /src/github.com/ipfs/go-ipfs,$(subst $(GOPATH_DELIMITER), ,$(GOPATH)))) - -deps: go_check gx_check path_check $(covertools_rule) - ${gx_bin} --verbose install --global - -deps_covertools: - go get -u github.com/wadey/gocovmerge - go get -u golang.org/x/tools/cmd/cover - -# saves/vendors third-party dependencies to Godeps/_workspace -# -r flag rewrites import paths to use the vendored path -# ./... performs operation on all packages in tree -vendor: godep - godep save -r ./... - -nofuse: GOTAGS += nofuse -nofuse: deps - $(MAKE) -C cmd/ipfs install - -install build: deps - $(MAKE) -C cmd/ipfs $@ - -clean: - $(MAKE) -C cmd/ipfs clean - $(MAKE) -C test clean - -uninstall: - $(MAKE) -C cmd/ipfs uninstall - -PHONY += all help godep gx_check covertools -PHONY += go_check deps vendor install build nofuse clean uninstall - -############################################################## -# tests targets - -test: test_expensive - -test_short: test_go_fmt build test_go_short test_sharness_short - -test_expensive: test_go_fmt build test_go_expensive test_sharness_expensive windows_build_check - -test_3node: - $(MAKE) -C test/3nodetest - -test_go_fmt: - bin/test-go-fmt - -test_go_short: GOTFLAGS += -test.short -test_go_race: GOTFLAGS += -race -test_go_expensive test_go_short test_go_race: - go test $(GOFLAGS) $(GOTFLAGS) ./... - -coverage: deps_covertools - @echo Running coverage - $(eval PKGS := $(shell go list -f '{{if (len .GoFiles)}}{{.ImportPath}}{{end}}' ./... | grep -v /vendor/ | grep -v /Godeps/)) -#$(eval PKGS_DELIM := $(call join-with,$(comma),$(PKGS))) - @go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test $(GOFLAGS) $(GOTFLAGS) -covermode=atomic -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile {{.ImportPath}}{{end}}' $(GOFLAGS) $(PKGS) | xargs -I {} bash -c {} 2>&1 | grep -v 'warning: no packages being tested depend on' - gocovmerge `ls *.coverprofile` > coverage.txt - rm *.coverprofile - bash -c 'bash <(curl -s https://codecov.io/bash)' - -test_sharness_short: - $(MAKE) -j1 -C test/sharness/ - -test_sharness_expensive: - TEST_EXPENSIVE=1 $(MAKE) -j1 -C test/sharness/ - -test_all_commits: - @echo "testing all commits between origin/master..HEAD" - @echo "WARNING: this will 'git rebase --exec'." - @test/bin/continueyn - GIT_EDITOR=true git rebase -i --exec "$(MAKE) test" origin/master - -test_all_commits_travis: - # these are needed because travis. - # we don't use this yet because it takes way too long. - git config --global user.email "nemo@ipfs.io" - git config --global user.name "IPFS BOT" - git fetch origin master:master - GIT_EDITOR=true git rebase -i --exec "$(MAKE) test" master - -# since we have CI for osx and linux but not windows, this should help -windows_build_check: - GOOS=windows GOARCH=amd64 go build -o .test.ipfs.exe ./cmd/ipfs - rm -f .test.ipfs.exe - -PHONY += test test_short test_expensive - -############################################################## -# A semi-helpful help message - -help: - @echo 'DEPENDENCY TARGETS:' - @echo '' - @echo ' gx_check - Installs or upgrades gx and gx-go' - @echo ' deps - Download dependencies using gx' - @echo ' vendor - Create a Godep workspace of 3rd party dependencies' - @echo '' - @echo 'BUILD TARGETS:' - @echo '' - @echo ' all - print this help message' - @echo ' build - Build binary at ./cmd/ipfs/ipfs' - @echo ' nofuse - Build binary with no fuse support' - @echo ' install - Build binary and install into $$GOPATH/bin' -# @echo ' dist_install - TODO: c.f. ./cmd/ipfs/dist/README.md' - @echo '' - @echo 'CLEANING TARGETS:' - @echo '' - @echo ' clean - Remove binary from build directory' - @echo ' uninstall - Remove binary from $$GOPATH/bin' - @echo '' - @echo 'TESTING TARGETS:' - @echo '' - @echo ' test - Run expensive tests and Window$$ check' - @echo ' test_short - Run short tests and sharness tests' - @echo ' test_expensive - Run a few extras' - @echo ' test_3node' - @echo ' test_go_short' - @echo ' test_go_expensive' - @echo ' test_go_race' - @echo ' test_sharness_short' - @echo ' test_sharness_expensive' - @echo ' test_all_commits' - @echo " test_all_commits_travis - DON'T USE: takes way too long" - @echo ' windows_build_check' - @echo '' - -PHONY += help - -.PHONY: $(PHONY) +include Rules.mk diff --git a/Rules.mk b/Rules.mk new file mode 100644 index 000000000..033086aa4 --- /dev/null +++ b/Rules.mk @@ -0,0 +1,137 @@ +TGT_BIN := +CLEAN := +COVERAGE := +DISTCLEAN := +TEST := +TEST_SHORT := + +all: help # all has to be first defined target +.PHONY: all + +include mk/util.mk +include mk/golang.mk +include mk/gx.mk + +# -------------------- # +# sub-files # +# -------------------- # +dir := bin +include $(dir)/Rules.mk + +dir := test +include $(dir)/Rules.mk + +dir := cmd/ipfs +include $(dir)/Rules.mk + +# has to be after cmd/ipfs due to PATH +dir := coverage +include $(dir)/Rules.mk + +dir := namesys/pb +include $(dir)/Rules.mk + +dir := unixfs/pb +include $(dir)/Rules.mk + +dir := merkledag/pb +include $(dir)/Rules.mk + +dir := exchange/bitswap/message/pb +include $(dir)/Rules.mk + +dir := diagnostics/pb +include $(dir)/Rules.mk + +dir := pin/internal/pb +include $(dir)/Rules.mk + +# -------------------- # +# universal rules # +# -------------------- # + +%.pb.go: %.proto + $(PROTOC) + +# -------------------- # +# extra properties # +# -------------------- # + +ifeq ($(TEST_NO_FUSE),1) + GOTAGS += nofuse +endif +export IPFS_REUSEPORT=false + +# -------------------- # +# core targets # +# -------------------- # + + +build: $(TGT_BIN) +.PHONY: build + +clean: + rm -f $(CLEAN) +.PHONY: clean + +coverage: $(COVERAGE) +.PHONY: coverage + +distclean: clean + rm -f $(DISTCLEAN) +.PHONY: distclean + +test: $(TEST) +.PHONY: test + +test_short: $(TEST_SHORT) +.PHONY: test_short + +deps: gx-deps +.PHONY: deps + +nofuse: GOTAGS += nofuse +nofuse: build +.PHONY: nofuse + +install: $$(DEPS_GO) + go install $(go-flags-with-tags) ./cmd/ipfs +.PHONY: install + +uninstall: + go clean -i ./cmd/ipfs +.PHONY: uninstall + +help: + @echo 'DEPENDENCY TARGETS:' + @echo '' + @echo ' deps - Download dependencies using bundled gx' + @echo ' test_sharness_deps - Download and build dependencies for sharness' + @echo '' + @echo 'BUILD TARGETS:' + @echo '' + @echo ' all - print this help message' + @echo ' build - Build binary at ./cmd/ipfs/ipfs' + @echo ' nofuse - Build binary with no fuse support' + @echo ' install - Build binary and install into $$GOPATH/bin' +# @echo ' dist_install - TODO: c.f. ./cmd/ipfs/dist/README.md' + @echo '' + @echo 'CLEANING TARGETS:' + @echo '' + @echo ' clean - Remove files generated by build' + @echo ' distclean - Remove files that are no part of a repository' + @echo ' uninstall - Remove binary from $$GOPATH/bin' + @echo '' + @echo 'TESTING TARGETS:' + @echo '' + @echo ' test - Run expensive tests' + @echo ' test_short - Run short tests and short sharness tests' + @echo ' test_go_short' + @echo ' test_go_expensive' + @echo ' test_go_race' + @echo ' test_sharness_short' + @echo ' test_sharness_expensive' + @echo ' test_sharness_race' + @echo ' coverage - Collects coverage info from unit tests and sharness' + @echo +.PHONY: help diff --git a/bin/Rules.mk b/bin/Rules.mk new file mode 100644 index 000000000..8744b3102 --- /dev/null +++ b/bin/Rules.mk @@ -0,0 +1,26 @@ +include mk/header.mk + +dist_root_$(d)=/ipfs/QmNZL8wNsvAGdVYr8uGeUE9aGfHjFpHegAWywQFEdSaJbp + +$(d)/gx: $(d)/gx-v0.9.0 +$(d)/gx-go: $(d)/gx-go-v1.3.0 + +TGTS_$(d) := $(d)/gx $(d)/gx-go +DISTCLEAN += $(wildcard $(d)/gx-v*) $(wildcard $(d)/gx-go-v*) + +PATH := $(realpath $(d)):$(PATH) + +$(TGTS_$(d)): + rm -f $@ + ln -s $(notdir $^) $@ + +bin/gx-v%: + @echo "installing gx $(@:bin/gx-%=%)" + bin/dist_get $(dist_root_bin) gx $@ $(@:bin/gx-%=%) + +bin/gx-go-v%: + @echo "installing gx-go $(@:bin/gx-go-%=%)" + @bin/dist_get $(dist_root_bin) gx-go $@ $(@:bin/gx-go-%=%) + +CLEAN += $(TGTS_$(d)) +include mk/footer.mk diff --git a/bin/gen-make-dep-chain b/bin/gen-make-dep-chain new file mode 100755 index 000000000..be00e1813 --- /dev/null +++ b/bin/gen-make-dep-chain @@ -0,0 +1,9 @@ +#!/bin/sh +(for p ; do + printf "%s\n" "$p" +done) | tac | +( read -r PREV ; +while read -r CURR ; do + printf '%s: | %s\n' "$PREV" "$CURR" + PREV="$CURR" +done) | tac diff --git a/circle.yml b/circle.yml index acf3aae44..c601afaf2 100644 --- a/circle.yml +++ b/circle.yml @@ -35,6 +35,6 @@ dependencies: test: override: - - case $CIRCLE_NODE_INDEX in 0) make coverage ;; 1) make test_sharness_expensive ;; esac: + - case $CIRCLE_NODE_INDEX in 0) make -j 1 coverage/unit_tests.coverprofile && bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile;; 1) make -j 1 coverage/sharness_tests.coverprofile && bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile;; esac: pwd: "../.go_workspace/src/$IMPORT_PATH" parallel: true diff --git a/cmd/ipfs/.gitignore b/cmd/ipfs/.gitignore index 264486b52..9393c5fb5 100644 --- a/cmd/ipfs/.gitignore +++ b/cmd/ipfs/.gitignore @@ -1,2 +1,3 @@ ipfs +ipfs-test-cover ipfs.exe diff --git a/cmd/ipfs/Makefile b/cmd/ipfs/Makefile deleted file mode 100644 index 59746c62e..000000000 --- a/cmd/ipfs/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -COMMIT := $(shell git rev-parse --short HEAD) -GOFLAGS += -ldflags="-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(COMMIT)" - -all: install - -install: - go install $(GOFLAGS) - -build: - go build -i $(GOFLAGS) - -clean: - go clean $(GOFLAGS) - -uninstall: - go clean -i $(GOFLAGS) diff --git a/cmd/ipfs/Rules.mk b/cmd/ipfs/Rules.mk new file mode 100644 index 000000000..7d6cf2b21 --- /dev/null +++ b/cmd/ipfs/Rules.mk @@ -0,0 +1,29 @@ +include mk/header.mk +IPFS_BIN_$(d) := $(call go-curr-pkg-tgt) + +TGT_BIN += $(IPFS_BIN_$(d)) +CLEAN += $(IPFS_BIN_$(d)) + +PATH := $(realpath $(d)):$(PATH) + +# disabled for now +# depend on *.pb.go files in the repo as Order Only (as they shouldn't be rebuilt if exist) +# DPES_OO_$(d) := diagnostics/pb/diagnostics.pb.go exchange/bitswap/message/pb/message.pb.go +# DEPS_OO_$(d) += merkledag/pb/merkledag.pb.go namesys/pb/namesys.pb.go +# DEPS_OO_$(d) += pin/internal/pb/header.pb.go unixfs/pb/unixfs.pb.go + +# uses second expansion to collect all $(DEPS_GO) +$(IPFS_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS #| $(DEPS_OO_$(d)) + $(go-build) + +COVER_BIN_$(d) := $(d)/ipfs-test-cover +CLEAN += $(COVER_BIN_$(d)) + +$(COVER_BIN_$(d)): GOTAGS += testrunmain +$(COVER_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS + $(eval TMP_PKGS := $(shell go list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) ./cmd/ipfs | sed 's/ /\n/g' | grep ipfs/go-ipfs)) + $(eval TMP_LIST := $(call join-with,$(comma),$(TMP_PKGS))) + @echo go test $@ -c -covermode atomic -coverpkg ... $(go-flags-with-tags) ./$(@D) # for info + @go test -o $@ -c -covermode atomic -coverpkg $(TMP_LIST) $(go-flags-with-tags) ./$(@D) 2>&1 | (grep -v 'warning: no packages being tested' || true) + +include mk/footer.mk diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index 9159ce7c5..bc48a5bf5 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -64,6 +64,10 @@ type cmdInvocation struct { // - output the response // - if anything fails, print error, maybe with help func main() { + os.Exit(mainRet()) +} + +func mainRet() int { rand.Seed(time.Now().UnixNano()) ctx := logging.ContextWithLoggable(context.Background(), loggables.Uuid("session")) var err error @@ -79,7 +83,7 @@ func main() { stopFunc, err := profileIfEnabled() if err != nil { printErr(err) - os.Exit(1) + return 1 } defer stopFunc() // to be executed as late as possible @@ -104,7 +108,7 @@ func main() { if len(os.Args) == 2 { if os.Args[1] == "help" { printHelp(false, os.Stdout) - os.Exit(0) + return 0 } else if os.Args[1] == "--version" { os.Args[1] = "version" } @@ -119,11 +123,11 @@ func main() { longH, shortH, err := invoc.requestedHelp() if err != nil { printErr(err) - os.Exit(1) + return 1 } if longH || shortH { printHelp(longH, os.Stdout) - os.Exit(0) + return 0 } } @@ -138,7 +142,7 @@ func main() { fmt.Fprintf(os.Stderr, "\n") printHelp(false, os.Stderr) } - os.Exit(1) + return 1 } // here we handle the cases where @@ -146,7 +150,7 @@ func main() { // - the main command is invoked. if invoc.cmd == nil || invoc.cmd.Run == nil { printHelp(false, os.Stdout) - os.Exit(0) + return 0 } // ok, finally, run the command invocation. @@ -161,16 +165,16 @@ func main() { if isClientError(err) { printMetaHelp(os.Stderr) } - os.Exit(1) + return 1 } // everything went better than expected :) _, err = io.Copy(os.Stdout, output) if err != nil { printErr(err) - - os.Exit(1) + return 1 } + return 0 } func (i *cmdInvocation) Run(ctx context.Context) (output io.Reader, err error) { diff --git a/cmd/ipfs/runmain_test.go b/cmd/ipfs/runmain_test.go new file mode 100644 index 000000000..8eecce334 --- /dev/null +++ b/cmd/ipfs/runmain_test.go @@ -0,0 +1,30 @@ +// +build testrunmain + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "testing" +) + +// this abuses go so much that I felt dirty writing this code +// but it is the only way to do it without writing custom compiler that would +// be a clone of go-build with go-test +func TestRunMain(t *testing.T) { + args := flag.Args() + os.Args = append([]string{os.Args[0]}, args...) + ret := mainRet() + + p := os.Getenv("IPFS_COVER_RET_FILE") + if len(p) != 0 { + ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", ret)), 0777) + } + + // close outputs so go testing doesn't print anything + null, _ := os.Open(os.DevNull) + os.Stderr = null + os.Stdout = null +} diff --git a/core/commands/config.go b/core/commands/config.go index f51576806..fd6aa194a 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -311,7 +311,7 @@ func editConfig(filename string) error { func replaceConfig(r repo.Repo, file io.Reader) error { var cfg config.Config if err := json.NewDecoder(file).Decode(&cfg); err != nil { - return errors.New("failed to decode file as config") + return errors.New("failed to decode file as config: " + err.Error()) } if len(cfg.Identity.PrivKey) != 0 { return errors.New("setting private key with API is not supported") diff --git a/coverage/.gitignore b/coverage/.gitignore new file mode 100644 index 000000000..bfe701a41 --- /dev/null +++ b/coverage/.gitignore @@ -0,0 +1,4 @@ +unitcover +sharnesscover +ipfs + diff --git a/coverage/Rules.mk b/coverage/Rules.mk new file mode 100644 index 000000000..ab2ebfd3a --- /dev/null +++ b/coverage/Rules.mk @@ -0,0 +1,51 @@ +include mk/header.mk + +$(d)/coverage_deps: + rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover + rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover + go get -u github.com/Kubuxu/gocovmerge + go get -u golang.org/x/tools/cmd/cover +.PHONY: $(d)/coverage_deps + +# unit tests coverage +UTESTS_$(d) := $(shell go list -f '{{if (len .TestGoFiles)}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps) + +UCOVER_$(d) := $(addsuffix .coverprofile,$(addprefix $(d)/unitcover/, $(subst /,_,$(UTESTS_$(d))))) + +$(UCOVER_$(d)): $(d)/coverage_deps ALWAYS + $(eval TMP_PKG := $(subst _,/,$(basename $(@F)))) + @go test $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=$@ $(TMP_PKG) + + +$(d)/unit_tests.coverprofile: $(UCOVER_$(d)) + gocovmerge $^ > $@ + +TGTS_$(d) := $(d)/unit_tests.coverprofile + + +# sharness tests coverage +$(d)/ipfs: GOTAGS += testrunmain +$(d)/ipfs: $(d)/main + $(go-build) + +CLEAN += $(d)/ipfs + +ifneq ($(filter coverage,$(MAKECMDGOALS)),) + # this is quite hacky but it is best way I could fiture out + DEPS_test/sharness += cmd/ipfs/ipfs-test-cover $(d)/coverage_deps $(d)/ipfs +endif + +export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ + +$(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short + (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ + + +PATH := $(realpath $(d)):$(PATH) + +TGTS_$(d) += $(d)/sharness_tests.coverprofile + +CLEAN += $(TGTS_$(d)) +COVERAGE += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/coverage/main/main.go b/coverage/main/main.go new file mode 100644 index 000000000..ba5d473e3 --- /dev/null +++ b/coverage/main/main.go @@ -0,0 +1,72 @@ +// +build testrunmain + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "os/signal" + "strconv" + "syscall" +) + +func main() { + coverDir := os.Getenv("IPFS_COVER_DIR") + if len(coverDir) == 0 { + fmt.Println("IPFS_COVER_DIR not defined") + os.Exit(1) + } + coverFile, err := ioutil.TempFile(coverDir, "coverage-") + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + retFile, err := ioutil.TempFile("", "cover-ret-file") + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + args := []string{"-test.run", "^TestRunMain$", "-test.coverprofile=" + coverFile.Name(), "--"} + args = append(args, os.Args[1:]...) + + p := exec.Command("ipfs-test-cover", args...) + p.Stdin = os.Stdin + p.Stdout = os.Stdout + p.Stderr = os.Stderr + p.Env = append(os.Environ(), "IPFS_COVER_RET_FILE="+retFile.Name()) + + p.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, + } + + sig := make(chan os.Signal, 1) + go func() { + for { + p.Process.Signal(<-sig) + } + }() + signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + + err = p.Run() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + b, err := ioutil.ReadAll(retFile) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + b = b[:len(b)-1] + d, err := strconv.Atoi(string(b)) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + os.Exit(d) +} diff --git a/diagnostics/pb/Makefile b/diagnostics/pb/Makefile deleted file mode 100644 index 334feee74..000000000 --- a/diagnostics/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/diagnostics/pb/Rules.mk b/diagnostics/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/diagnostics/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/exchange/bitswap/message/pb/Makefile b/exchange/bitswap/message/pb/Makefile deleted file mode 100644 index 5bbebea07..000000000 --- a/exchange/bitswap/message/pb/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# TODO(brian): add proto tasks -all: message.pb.go - -message.pb.go: message.proto - protoc --gogo_out=. --proto_path=../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm message.pb.go diff --git a/exchange/bitswap/message/pb/Rules.mk b/exchange/bitswap/message/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/exchange/bitswap/message/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/merkledag/pb/Makefile b/merkledag/pb/Makefile deleted file mode 100644 index 08ac883d0..000000000 --- a/merkledag/pb/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm -f *.pb.go - rm -f *.go diff --git a/merkledag/pb/Rules.mk b/merkledag/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/merkledag/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/mk/footer.mk b/mk/footer.mk new file mode 100644 index 000000000..4224aaa97 --- /dev/null +++ b/mk/footer.mk @@ -0,0 +1,2 @@ +d := $(dirstack_$(sp)) +sp := $(basename $(sp)) diff --git a/mk/golang.mk b/mk/golang.mk new file mode 100644 index 000000000..c0cc130a2 --- /dev/null +++ b/mk/golang.mk @@ -0,0 +1,50 @@ +# golang utilities +GO_MIN_VERSION = 1.7 + +# pre-definitions +GOTAGS ?= +GOFLAGS ?= +GOTFLAGS ?= + +DEPS_GO := +TEST_GO := +CHECK_GO := + +go-pkg-name=$(shell go list $(go-tags) ./$(1)) +go-main-name=$(notdir $(call go-pkg-name,$(1)))$(?exe) +go-curr-pkg-tgt=$(d)/$(call go-main-name,$(d)) + +go-tags=$(if $(GOTAGS), -tags $(call join-with,$(comma),$(GOTAGS))) +go-flags-with-tags=$(GOFLAGS)$(go-tags) + +define go-build +go build -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$<)" +endef + +test_go_short: GOTFLAGS += -test.short +test_go_short: test_go_expensive +.PHONY: test_go_short + +test_go_race: GOTFLAGS += -race +test_go_race: test_go_expensive +.PHONY: test_go_race + +test_go_expensive: $$(DEPS_GO) + go test $(go-flags-with-tags) $(GOTFLAGS) ./... +.PHONY: test_go_expensive +TEST_GO += test_go_expensive + +test_go_fmt: + bin/test-go-fmt +.PHONY: test_go_fmt +TEST_GO += test_go_fmt + +test_go: $(TEST_GO) + +check_go_version: + bin/check_go_version $(GO_MIN_VERSION) +.PHONY: check_go_version +CHECK_GO += check_go_version + +TEST += $(TEST_GO) +TEST_SHORT += test_go_fmt test_go_short diff --git a/mk/gx.mk b/mk/gx.mk new file mode 100644 index 000000000..d2f70f59a --- /dev/null +++ b/mk/gx.mk @@ -0,0 +1,6 @@ +gx-path = gx/ipfs/$(shell gx deps find $(1))/$(1) + +gx-deps: bin/gx bin/gx-go $(CHECK_GO) + gx install --global >/dev/null 2>&1 + +DEPS_GO += gx-deps diff --git a/mk/header.mk b/mk/header.mk new file mode 100644 index 000000000..fc60061e1 --- /dev/null +++ b/mk/header.mk @@ -0,0 +1,4 @@ +# keep track of dirs +p := $(sp).x +dirstack_$(sp) := $(d) +d := $(dir) diff --git a/mk/util.mk b/mk/util.mk new file mode 100644 index 000000000..8f76013df --- /dev/null +++ b/mk/util.mk @@ -0,0 +1,20 @@ +# util functions +ifeq ($(OS),Windows_NT) + WINDOWS :=1 + ?exe :=.exe # windows compat +else + ?exe := +endif + +space:= +space+= +comma:=, +join-with=$(subst $(space),$1,$(strip $2)) + +# debug target, prints varaible. Example: `make print-GOFLAGS` +print-%: + @echo $*=$($*) + +# phony target that will mean that recipe is always exectued +ALWAYS: +.PHONY: ALWAYS diff --git a/namesys/pb/Makefile b/namesys/pb/Makefile deleted file mode 100644 index 334feee74..000000000 --- a/namesys/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/namesys/pb/Rules.mk b/namesys/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/namesys/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/pin/internal/pb/Rules.mk b/pin/internal/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/pin/internal/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/pin/pin.go b/pin/pin.go index baf0d5958..c0eccc203 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -5,7 +5,6 @@ package pin import ( "context" "fmt" - "os" "sync" "time" @@ -26,8 +25,9 @@ var emptyKey *cid.Cid func init() { e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) + msg := "failed to decode empty key constant" + log.Error(msg) + panic(msg) } emptyKey = e } diff --git a/test/Makefile b/test/Makefile deleted file mode 100644 index d7a320e58..000000000 --- a/test/Makefile +++ /dev/null @@ -1,115 +0,0 @@ - -BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep -BINS += bin/go-timeout -IPFS_ROOT = ../ -IPFS_CMD = ../cmd/ipfs -RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random -RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files -POLLENDPOINT_SRC= ../thirdparty/pollEndpoint -GOSLEEP_SRC = ./dependencies/go-sleep -GOTIMEOUT_SRC = ./dependencies/go-timeout - -export PATH := ../bin:${PATH} - -# User might want to override those on the command line -GOFLAGS = - -all: deps - -global-deps: - $(MAKE) -C .. deps - -deps: global-deps bins - -clean: - $(MAKE) -C sharness clean - rm -rf $(BINS) - -bins: $(BINS) - -find_go_files = $(shell find $(1) -name "*.go") - -# Non gx dependencies - -bin/random: $(call find_go_files, $(RANDOM_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/random $(RANDOM_SRC)/random - -bin/random-files: - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/random-files $(RANDOM_FILES_SRC)/random-files - -bin/ipfs: $(call find_go_files, $(IPFS_ROOT)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/ipfs $(IPFS_CMD) - -bin/pollEndpoint: $(call find_go_files, $(POLLENDPOINT_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/pollEndpoint $(POLLENDPOINT_SRC) - -bin/go-sleep: $(call find_go_files, $(GOSLEEP_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/go-sleep $(GOSLEEP_SRC) - -bin/go-timeout: $(call find_go_files, $(GOTIMEOUT_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/go-timeout $(GOTIMEOUT_SRC) - -# gx dependencies - -multihash_src: - $(eval MULTIHASH_HASH := $(shell cd .. && bin/gx deps find go-multihash)) - $(eval MULTIHASH_SRC := gx/ipfs/$(MULTIHASH_HASH)/go-multihash) - -hang-fds_src: - $(eval HANG_FDS_HASH := $(shell cd .. && bin/gx deps find hang-fds)) - $(eval HANG_FDS_SRC := gx/ipfs/$(HANG_FDS_HASH)/hang-fds) - -bin/multihash: multihash_src $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash - -bin/hang-fds: hang-fds_src $(call find_go_files, $(HANG_FDS_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/hang-fds $(HANG_FDS_SRC) - -iptb_src: - $(eval IPTB_HASH := $(shell cd .. && bin/gx deps find iptb)) - $(eval IPTB_SRC := gx/ipfs/$(IPTB_HASH)/iptb) - -bin/iptb: iptb_src $(call find_go_files, $(IPTB_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/iptb $(IPTB_SRC) - -# Tests - -test: test_expensive - -test_expensive: verify_gofmt - $(MAKE) -C sharness TEST_EXPENSIVE=1 - $(MAKE) -C 3nodetest - $(MAKE) -C dependencies - -test_cheap: verify_gofmt - $(MAKE) -C sharness - $(MAKE) -C 3nodetest - -test_race: verify_gofmt - $(MAKE) -C sharness GOFLAGS=-race TEST_EXPENSIVE=1 - $(MAKE) -C 3nodetest GOFLAGS=-race - $(MAKE) -C dependencies GOFLAGS=-race - -# Misc - -coverage: coverage_sharness - -coverage_sharness: - ./sharness_test_coverage_helper.sh - -IPFS-BUILD-OPTIONS: FORCE - @bin/checkflags '$@' '$(GOFLAGS)' '*** new Go flags ***' - -verify_gofmt: - bin/verify-go-fmt.sh - -.PHONY: all clean FORCE diff --git a/test/Rules.mk b/test/Rules.mk new file mode 100644 index 000000000..e596dfd55 --- /dev/null +++ b/test/Rules.mk @@ -0,0 +1,9 @@ +include mk/header.mk + +dir := $(d)/bin +include $(dir)/Rules.mk + +dir := $(d)/sharness +include $(dir)/Rules.mk + +include mk/footer.mk diff --git a/test/bin/.gitignore b/test/bin/.gitignore index c032badee..cf2d84738 100644 --- a/test/bin/.gitignore +++ b/test/bin/.gitignore @@ -8,3 +8,4 @@ !checkflags !continueyn !verify-go-fmt.sh +!Rules.mk diff --git a/test/bin/Rules.mk b/test/bin/Rules.mk new file mode 100644 index 000000000..08f1a295c --- /dev/null +++ b/test/bin/Rules.mk @@ -0,0 +1,44 @@ +include mk/header.mk + +TGTS_$(d) := + +$(d)/random: Godeps/_workspace/src/github.com/jbenet/go-random/random + $(go-build) +TGTS_$(d) += $(d)/random + +$(d)/random-files: Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files + $(go-build) +TGTS_$(d) += $(d)/random-files + +$(d)/pollEndpoint: thirdparty/pollEndpoint + $(go-build) +TGTS_$(d) += $(d)/pollEndpoint + +$(d)/go-sleep: test/dependencies/go-sleep + $(go-build) +TGTS_$(d) += $(d)/go-sleep + +$(d)/go-timeout: test/dependencies/go-timeout + $(go-build) +TGTS_$(d) += $(d)/go-timeout + +TGTS_GX_$(d) := hang-fds iptb +TGTS_GX_$(d) := $(addprefix $(d)/,$(TGTS_GX_$(d))) + +$(TGTS_GX_$(d)): + go build -i $(go-flags-with-tags) -o "$@" "$(call gx-path,$(notdir $@))" + +TGTS_$(d) += $(TGTS_GX_$(d)) + +# multihash is special +$(d)/multihash: + go build -i $(go-flags-with-tags) -o "$@" "gx/ipfs/$(shell gx deps find go-multihash)/go-multihash/multihash" +TGTS_$(d) += $(d)/multihash + +$(TGTS_$(d)): $$(DEPS_GO) + +CLEAN += $(TGTS_$(d)) + +PATH := $(realpath $(d)):$(PATH) + +include mk/footer.mk diff --git a/test/sharness/Makefile b/test/sharness/Makefile deleted file mode 100644 index 922e1ecbb..000000000 --- a/test/sharness/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -# Run tests -# -# Copyright (c) 2014 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# - -# NOTE: Run with TEST_VERBOSE=1 for verbose sharness tests. - -T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) -BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \ - bin/iptb bin/go-sleep bin/random-files bin/go-timeout bin/hang-fds -SHARNESS = lib/sharness/sharness.sh -IPFS_ROOT = ../.. - -# User might want to override those on the command line -GOFLAGS = - -all: aggregate - -clean: clean-test-results - @echo "*** $@ ***" - rm -rf $(BINS) - -clean-test-results: - @echo "*** $@ ***" - rm -rf test-results - -$(T): clean-test-results deps - @echo "*** $@ ***" - ./$@ - -aggregate: clean-test-results $(T) - @echo "*** $@ ***" - lib/test-aggregate-results.sh - -global-deps: - $(MAKE) -C ../.. deps - -deps: global-deps $(SHARNESS) $(BINS) curl - -$(SHARNESS): FORCE - @echo "*** checking $@ ***" - lib/install-sharness.sh - -bin/%: FORCE - $(MAKE) -C .. GOFLAGS=$(GOFLAGS) $@ - -race: - $(MAKE) GOFLAGS=-race all - -.PHONY: all clean $(T) aggregate FORCE - -# will fail if curl is not installed. -# TODO: get rid of this and install curl with git or ipfs. -install_curl_pls = "curl is required to run tests. please install it" -curl: - @which curl >/dev/null || (echo "$(install_curl_pls)" && false) diff --git a/test/sharness/README.md b/test/sharness/README.md index 910fa6e0d..39291d0c5 100644 --- a/test/sharness/README.md +++ b/test/sharness/README.md @@ -37,7 +37,8 @@ $ ./t0010-basic-commands.sh -v -i ## Sharness -When running "make" in this directory for the first time, sharness +When running sharness tests from main Makefile or when `test_sharness_deps` +target is run dependencies for sharness will be downloaded from its github repo and installed in a "lib/sharness" directory. diff --git a/test/sharness/Rules.mk b/test/sharness/Rules.mk new file mode 100644 index 000000000..ff109da09 --- /dev/null +++ b/test/sharness/Rules.mk @@ -0,0 +1,54 @@ +include mk/header.mk + + +SHARNESS_$(d) = $(d)/lib/sharness/sharness.sh + +T_$(d) = $(sort $(wildcard $(d)/t[0-9][0-9][0-9][0-9]-*.sh)) + +DEPS_$(d) := test/bin/random test/bin/multihash test/bin/pollEndpoint \ + test/bin/iptb test/bin/go-sleep test/bin/random-files \ + test/bin/go-timeout test/bin/hang-fds +DEPS_$(d) += cmd/ipfs/ipfs +DEPS_$(d) += $(d)/clean-test-results +DEPS_$(d) += $(SHARNESS_$(d)) + +export MAKE_SKIP_PATH=1 + +$(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency + @echo "*** $@ ***" + @(cd $(@D) && ./$(@F)) 2>&1 +.PHONY: $(T_$(d)) + +$(d)/aggregate: $(T_$(d)) + @echo "*** $@ ***" + @(cd $(@D) && ./lib/test-aggregate-results.sh) +.PHONY: $(d)/aggregate + +$(d)/clean-test-results: + rm -rf $(@D)/test-results +.PHONY: $(d)/clean-test-results + +CLEAN += $(wildcard $(d)/test-results/*) + +$(SHARNESS_$(d)): $(d) ALWAYS + @clonedir=$(dir $(@D)) $&2 "Cannot find the tests' local ipfs tool." + echo >&2 "Please check test and ipfs tool installation." + exit 1 + fi +fi # set sharness verbosity. we set the env var directly as # it's too late to pass in --verbose, and --verbose is harder # to pass through in some cases. test "$TEST_VERBOSE" = 1 && verbose=t - -# assert the `ipfs` we're using is the right one. -if test `which ipfs` != ${BIN}/ipfs; then - echo >&2 "Cannot find the tests' local ipfs tool." - echo >&2 "Please check test and ipfs tool installation." - exit 1 -fi - - # source the common hashes first. . lib/test-lib-hashes.sh diff --git a/test/sharness/t0600-issues-and-regressions-online.sh b/test/sharness/t0600-issues-and-regressions-online.sh index 7499b0e1d..f5e5b8940 100755 --- a/test/sharness/t0600-issues-and-regressions-online.sh +++ b/test/sharness/t0600-issues-and-regressions-online.sh @@ -37,7 +37,8 @@ test_kill_ipfs_daemon test_expect_success "ipfs daemon --offline --mount fails - #2995" ' test_expect_code 1 ipfs daemon --offline --mount 2>daemon_err && - grep "mount is not currently supported in offline mode" daemon_err + grep "mount is not currently supported in offline mode" daemon_err || + test_fsh cat daemon_err ' test_done diff --git a/unixfs/pb/Makefile b/unixfs/pb/Makefile deleted file mode 100644 index 334feee74..000000000 --- a/unixfs/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/unixfs/pb/Rules.mk b/unixfs/pb/Rules.mk new file mode 100644 index 000000000..505f70e75 --- /dev/null +++ b/unixfs/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk