From 3eb614fcb29823f3c2c5cc796b47397a8d081cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 28 Dec 2017 20:11:28 +0100 Subject: [PATCH] sharness: Generate JUnit test reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- test/sharness/Rules.mk | 5 + .../0001-Generate-partial-JUnit-reports.patch | 200 ++++++++++++++++++ test/sharness/lib/gen-junit-report.sh | 8 + test/sharness/lib/install-sharness.sh | 15 +- test/sharness/lib/test-lib.sh | 2 +- test/sharness/t0181-private-network.sh | 4 +- 6 files changed, 229 insertions(+), 5 deletions(-) create mode 100644 test/sharness/lib/0001-Generate-partial-JUnit-reports.patch create mode 100755 test/sharness/lib/gen-junit-report.sh diff --git a/test/sharness/Rules.mk b/test/sharness/Rules.mk index e98e2c782..9cddc66fa 100644 --- a/test/sharness/Rules.mk +++ b/test/sharness/Rules.mk @@ -29,12 +29,17 @@ export MAKE_SKIP_PATH=1 $(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency @echo "*** $@ ***" +ifeq ($(CONTINUE_ON_S_FAILURE),1) + -@(cd $(@D) && ./$(@F)) 2>&1 +else @(cd $(@D) && ./$(@F)) 2>&1 +endif .PHONY: $(T_$(d)) $(d)/aggregate: $(T_$(d)) @echo "*** $@ ***" @(cd $(@D) && ./lib/test-aggregate-results.sh) + @(cd $(@D) && ./lib/gen-junit-report.sh) .PHONY: $(d)/aggregate $(d)/clean-test-results: diff --git a/test/sharness/lib/0001-Generate-partial-JUnit-reports.patch b/test/sharness/lib/0001-Generate-partial-JUnit-reports.patch new file mode 100644 index 000000000..be5bf7f83 --- /dev/null +++ b/test/sharness/lib/0001-Generate-partial-JUnit-reports.patch @@ -0,0 +1,200 @@ +From 0e2a489651e8e2b1aa4abf1cb2587fc3d0f78ce6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Magiera?= +Date: Thu, 28 Dec 2017 19:24:55 +0100 +Subject: [PATCH] Generate partial JUnit reports + +--- + sharness.sh | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- + 1 file changed, 73 insertions(+), 6 deletions(-) + +diff --git a/sharness.sh b/sharness.sh +index 6750ff7..55e10ac 100644 +--- a/sharness.sh ++++ b/sharness.sh +@@ -1,4 +1,4 @@ +-#!/bin/sh ++#!/usr/bin/env bash + # + # Copyright (c) 2011-2012 Mathias Lafeldt + # Copyright (c) 2005-2012 Git project +@@ -106,6 +106,8 @@ if test -n "$color"; then + test -n "$quiet" && return;; + esac + shift ++ ++ echo "$*" >> .junit/tout + printf "%s" "$*" + tput sgr0 + echo +@@ -115,6 +117,8 @@ else + say_color() { + test -z "$1" && test -n "$quiet" && return + shift ++ ++ echo "$*" >> .junit/tout + printf "%s\n" "$*" + } + fi +@@ -129,6 +133,12 @@ say() { + say_color info "$*" + } + ++esc=$(printf '\033') ++ ++esc_xml() { ++ sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"$esc"'/\'/g; s///g;' ++} ++ + test -n "$test_description" || error "Test script did not set test_description." + + if test "$help" = "t"; then +@@ -251,30 +261,69 @@ test_have_prereq() { + test $total_prereq = $ok_prereq + } + ++junit_testcase() { ++ test_name=$1 ++ tc_file=".junit/case-$(printf "%04d" $test_count)" ++ ++ shift ++ cat > "$tc_file" <<-EOF ++ ++ $@ ++ EOF ++ ++ if test -f .junit/tout; then ++ cat >> "$tc_file" <<-EOF ++ ++ $(cat .junit/tout | esc_xml) ++ ++ EOF ++ fi ++ ++ if test -f .junit/terr; then ++ cat >> "$tc_file" <<-EOF ++ ++ $(cat .junit/terr | esc_xml) ++ ++ EOF ++ fi ++ ++ echo "" >> "$tc_file" ++ rm -f .junit/tout .junit/terr ++} ++ + # You are not expected to call test_ok_ and test_failure_ directly, use + # the text_expect_* functions instead. + + test_ok_() { + test_success=$(($test_success + 1)) + say_color "" "ok $test_count - $@" ++ ++ junit_testcase "$@" + } + + test_failure_() { + test_failure=$(($test_failure + 1)) + say_color error "not ok $test_count - $1" ++ test_name=$1 + shift + echo "$@" | sed -e 's/^/# /' ++ junit_testcase "$test_name" ''$(echo $@ | esc_xml)'' ++ + test "$immediate" = "" || { EXIT_OK=t; exit 1; } + } + + test_known_broken_ok_() { + test_fixed=$(($test_fixed + 1)) + say_color error "ok $test_count - $@ # TODO known breakage vanished" ++ ++ junit_testcase "$@" '' + } + + test_known_broken_failure_() { + test_broken=$(($test_broken + 1)) + say_color warn "not ok $test_count - $@ # TODO known breakage" ++ ++ junit_testcase "$@" + } + + # Public: Execute commands in debug mode. +@@ -310,7 +359,7 @@ test_pause() { + test_eval_() { + # This is a separate function because some tests use + # "return" to end a test_expect_success block early. +- eval &3 2>&4 "$*" ++ eval >(tee -a .junit/tout >&3) 2> >(tee -a .junit/terr >&4) "$*" + } + + test_run_() { +@@ -355,8 +404,16 @@ test_skip_() { + of_prereq=" of $test_prereq" + fi + +- say_color skip >&3 "skipping test: $@" +- say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" ++ say_color skip >&3 "skipping test: $1" ++ say_color skip "ok $test_count # skip $1 (missing $missing_prereqm${of_prereq})" ++ ++ cat > ".junit/case-$(printf "%04d" $test_count)" <<-EOF ++ ++ ++ skip $(echo $1 | esc_xml) (missing $missing_prereq${of_prereq}) ++ ++ ++ EOF + : true + ;; + *) +@@ -403,7 +460,7 @@ test_expect_success() { + test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= + test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_success" + export test_prereq +- if ! test_skip_ "$@"; then ++ if ! test_skip_ "$@" "$1"; then + say >&3 "expecting success: $2" + if test_run_ "$2"; then + test_ok_ "$1" +@@ -442,7 +499,7 @@ test_expect_failure() { + test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= + test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_failure" + export test_prereq +- if ! test_skip_ "$@"; then ++ if ! test_skip_ "$@" "$1"; then + say >&3 "checking known breakage: $2" + if test_run_ "$2" expecting_failure; then + test_known_broken_ok_ "$1" +@@ -675,6 +732,7 @@ test_done() { + test_results_dir="$SHARNESS_TEST_DIRECTORY/test-results" + mkdir -p "$test_results_dir" + test_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.counts" ++ junit_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.xml.part" + + cat >>"$test_results_path" <<-EOF + total $test_count +@@ -684,6 +742,12 @@ test_done() { + failed $test_failure + + EOF ++ ++ cat >>"$junit_results_path" <<-EOF ++ ++ $(find .junit -name 'case-*' | sort | xargs -i cat "{}") ++ ++ EOF + fi + + if test "$test_fixed" != 0; then +@@ -771,6 +835,9 @@ mkdir -p "$test_dir" || exit 1 + # in subprocesses like git equals our $PWD (for pathname comparisons). + cd -P "$test_dir" || exit 1 + ++# Prepare JUnit report dir ++mkdir -p .junit ++ + this_test=${SHARNESS_TEST_FILE##*/} + this_test=${this_test%.$SHARNESS_TEST_EXTENSION} + for skp in $SKIP_TESTS; do +-- +2.15.1 + diff --git a/test/sharness/lib/gen-junit-report.sh b/test/sharness/lib/gen-junit-report.sh new file mode 100755 index 000000000..befb9db8b --- /dev/null +++ b/test/sharness/lib/gen-junit-report.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +cat > test-results/sharness.xml <<-EOF + + + $(find test-results -name '*.xml.part' | sort | xargs -i cat "{}") + +EOF diff --git a/test/sharness/lib/install-sharness.sh b/test/sharness/lib/install-sharness.sh index d2e93ece9..56b2a8132 100755 --- a/test/sharness/lib/install-sharness.sh +++ b/test/sharness/lib/install-sharness.sh @@ -7,13 +7,15 @@ # settings version=5eee9b51b5621cec95a64018f0cc779963b230d2 +patch_version=1 + urlprefix=https://github.com/mlafeldt/sharness.git if test ! -n "$clonedir" ; then clonedir=lib fi sharnessdir=sharness -if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version" +if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_${version}_p${patch_version}" then # There is the right version file. Great, we are done! exit 0 @@ -24,11 +26,20 @@ die() { exit 1 } +apply_patches() { + git config --local user.email "noone@nowhere" + git config --local user.name "No One" + git am ../0001-Generate-partial-JUnit-reports.patch + + touch "SHARNESS_VERSION_${version}_p${patch_version}" || die "Could not create 'SHARNESS_VERSION_${version}_p${patch_version}'" +} + checkout_version() { git checkout "$version" || die "Could not checkout '$version'" rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'" - touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'" echo "Sharness version $version is checked out!" + + apply_patches } if test -d "$clonedir/$sharnessdir/.git" diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 924f8d94f..d5ce02eba 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -214,6 +214,7 @@ test_launch_ipfs_daemon() { test_expect_success "'ipfs daemon' succeeds" ' ipfs daemon $args >actual_daemon 2>daemon_err & + IPFS_PID=$! ' # wait for api file to show up @@ -225,7 +226,6 @@ test_launch_ipfs_daemon() { # we say the daemon is ready when the API server is ready. test_expect_success "'ipfs daemon' is ready" ' - IPFS_PID=$! && pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' diff --git a/test/sharness/t0181-private-network.sh b/test/sharness/t0181-private-network.sh index cf2efc6b6..3e0e65148 100755 --- a/test/sharness/t0181-private-network.sh +++ b/test/sharness/t0181-private-network.sh @@ -29,7 +29,7 @@ pnet_key() { random 16 } -pnet_key > $IPFS_PATH/swarm.key +pnet_key > "${IPFS_PATH}/swarm.key" LIBP2P_FORCE_PNET=1 test_launch_ipfs_daemon @@ -42,7 +42,7 @@ set_key() { node="$1" keyfile="$2" - cp "$keyfile" "$IPTB_ROOT/$node/swarm.key" + cp "$keyfile" "${IPTB_ROOT}/${node}/swarm.key" } pnet_key > key1