sharness: Generate JUnit test reports

License: MIT
Signed-off-by: Łukasz Magiera <magik6k@gmail.com>

test/sharness/Rules.mk

@@ -29,12 +29,17 @@ export MAKE_SKIP_PATH=1
 
 $(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency
 	@echo "*** $@ ***"
+ifeq ($(CONTINUE_ON_S_FAILURE),1)
+	-@(cd $(@D) && ./$(@F)) 2>&1
+else
 	@(cd $(@D) && ./$(@F)) 2>&1
+endif
 .PHONY: $(T_$(d))
 
 $(d)/aggregate: $(T_$(d))
 	@echo "*** $@ ***"
 	@(cd $(@D) && ./lib/test-aggregate-results.sh)
+	@(cd $(@D) && ./lib/gen-junit-report.sh)
 .PHONY: $(d)/aggregate
 
 $(d)/clean-test-results:

test/sharness/lib/0001-Generate-partial-JUnit-reports.patch

@@ -0,0 +1,200 @@
+From 0e2a489651e8e2b1aa4abf1cb2587fc3d0f78ce6 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
+Date: Thu, 28 Dec 2017 19:24:55 +0100
+Subject: [PATCH] Generate partial JUnit reports
+
+---
+ sharness.sh | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
+ 1 file changed, 73 insertions(+), 6 deletions(-)
+
+diff --git a/sharness.sh b/sharness.sh
+index 6750ff7..55e10ac 100644
+--- a/sharness.sh
++++ b/sharness.sh
+@@ -1,4 +1,4 @@
+-#!/bin/sh
++#!/usr/bin/env bash
+ #
+ # Copyright (c) 2011-2012 Mathias Lafeldt
+ # Copyright (c) 2005-2012 Git project
+@@ -106,6 +106,8 @@ if test -n "$color"; then
+ 			test -n "$quiet" && return;;
+ 		esac
+ 		shift
++
++		echo "$*" >> .junit/tout
+ 		printf "%s" "$*"
+ 		tput sgr0
+ 		echo
+@@ -115,6 +117,8 @@ else
+ 	say_color() {
+ 		test -z "$1" && test -n "$quiet" && return
+ 		shift
++
++		echo "$*" >> .junit/tout
+ 		printf "%s\n" "$*"
+ 	}
+ fi
+@@ -129,6 +133,12 @@ say() {
+ 	say_color info "$*"
+ }
+ 
++esc=$(printf '\033')
++
++esc_xml() {
++	sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g; s/'"$esc"'/\&#39;/g; s///g;'
++}
++
+ test -n "$test_description" || error "Test script did not set test_description."
+ 
+ if test "$help" = "t"; then
+@@ -251,30 +261,69 @@ test_have_prereq() {
+ 	test $total_prereq = $ok_prereq
+ }
+ 
++junit_testcase() {
++	test_name=$1
++	tc_file=".junit/case-$(printf "%04d" $test_count)"
++
++	shift
++	cat > "$tc_file" <<-EOF
++	<testcase name="$test_count - $(echo $test_name | esc_xml) ">
++	$@
++	EOF
++
++	if test -f .junit/tout; then
++		cat >> "$tc_file" <<-EOF
++		<system-out>
++			$(cat .junit/tout | esc_xml)
++		</system-out>
++		EOF
++	fi
++
++	if test -f .junit/terr; then
++		cat >> "$tc_file" <<-EOF
++		<system-err>
++			$(cat .junit/terr | esc_xml)
++		</system-err>
++		EOF
++	fi
++
++	echo "</testcase>" >> "$tc_file"
++	rm -f .junit/tout .junit/terr
++}
++
+ # You are not expected to call test_ok_ and test_failure_ directly, use
+ # the text_expect_* functions instead.
+ 
+ test_ok_() {
+ 	test_success=$(($test_success + 1))
+ 	say_color "" "ok $test_count - $@"
++
++	junit_testcase "$@"
+ }
+ 
+ test_failure_() {
+ 	test_failure=$(($test_failure + 1))
+ 	say_color error "not ok $test_count - $1"
++	test_name=$1
+ 	shift
+ 	echo "$@" | sed -e 's/^/#	/'
++	junit_testcase "$test_name" '<failure type="">'$(echo $@ | esc_xml)'</failure>'
++
+ 	test "$immediate" = "" || { EXIT_OK=t; exit 1; }
+ }
+ 
+ test_known_broken_ok_() {
+ 	test_fixed=$(($test_fixed + 1))
+ 	say_color error "ok $test_count - $@ # TODO known breakage vanished"
++
++	junit_testcase "$@" '<failure type="known breakage vanished"/>'
+ }
+ 
+ test_known_broken_failure_() {
+ 	test_broken=$(($test_broken + 1))
+ 	say_color warn "not ok $test_count - $@ # TODO known breakage"
++
++	junit_testcase "$@"
+ }
+ 
+ # Public: Execute commands in debug mode.
+@@ -310,7 +359,7 @@ test_pause() {
+ test_eval_() {
+ 	# This is a separate function because some tests use
+ 	# "return" to end a test_expect_success block early.
+-	eval </dev/null >&3 2>&4 "$*"
++	eval </dev/null > >(tee -a .junit/tout >&3) 2> >(tee -a .junit/terr >&4) "$*"
+ }
+ 
+ test_run_() {
+@@ -355,8 +404,16 @@ test_skip_() {
+ 			of_prereq=" of $test_prereq"
+ 		fi
+ 
+-		say_color skip >&3 "skipping test: $@"
+-		say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})"
++		say_color skip >&3 "skipping test: $1"
++		say_color skip "ok $test_count # skip $1 (missing $missing_prereqm${of_prereq})"
++
++		cat > ".junit/case-$(printf "%04d" $test_count)" <<-EOF
++		<testcase name="$test_count - $(echo $2 | esc_xml)">
++			<skipped>
++				skip $(echo $1 | esc_xml) (missing $missing_prereq${of_prereq})
++			</skipped>
++		</testcase>
++		EOF
+ 		: true
+ 		;;
+ 	*)
+@@ -403,7 +460,7 @@ test_expect_success() {
+ 	test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
+ 	test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_success"
+ 	export test_prereq
+-	if ! test_skip_ "$@"; then
++	if ! test_skip_ "$@" "$1"; then
+ 		say >&3 "expecting success: $2"
+ 		if test_run_ "$2"; then
+ 			test_ok_ "$1"
+@@ -442,7 +499,7 @@ test_expect_failure() {
+ 	test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
+ 	test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_failure"
+ 	export test_prereq
+-	if ! test_skip_ "$@"; then
++	if ! test_skip_ "$@" "$1"; then
+ 		say >&3 "checking known breakage: $2"
+ 		if test_run_ "$2" expecting_failure; then
+ 			test_known_broken_ok_ "$1"
+@@ -675,6 +732,7 @@ test_done() {
+ 		test_results_dir="$SHARNESS_TEST_DIRECTORY/test-results"
+ 		mkdir -p "$test_results_dir"
+ 		test_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.counts"
++		junit_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.xml.part"
+ 
+ 		cat >>"$test_results_path" <<-EOF
+ 		total $test_count
+@@ -684,6 +742,12 @@ test_done() {
+ 		failed $test_failure
+ 
+ 		EOF
++
++		cat >>"$junit_results_path" <<-EOF
++		<testsuite errors="$test_broken" failures="$((test_failure+test_fixed))" tests="$test_count" name="$SHARNESS_TEST_FILE">
++			$(find .junit -name 'case-*' | sort | xargs -i cat "{}")
++		</testsuite>
++		EOF
+ 	fi
+ 
+ 	if test "$test_fixed" != 0; then
+@@ -771,6 +835,9 @@ mkdir -p "$test_dir" || exit 1
+ # in subprocesses like git equals our $PWD (for pathname comparisons).
+ cd -P "$test_dir" || exit 1
+ 
++# Prepare JUnit report dir
++mkdir -p .junit
++
+ this_test=${SHARNESS_TEST_FILE##*/}
+ this_test=${this_test%.$SHARNESS_TEST_EXTENSION}
+ for skp in $SKIP_TESTS; do
+-- 
+2.15.1
+

test/sharness/lib/gen-junit-report.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+cat > test-results/sharness.xml <<-EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="sharness">
+  $(find test-results -name '*.xml.part' | sort | xargs -i cat "{}")
+</testsuites>
+EOF

test/sharness/lib/install-sharness.sh

@@ -7,13 +7,15 @@
 
 # settings
 version=5eee9b51b5621cec95a64018f0cc779963b230d2
+patch_version=1
+
 urlprefix=https://github.com/mlafeldt/sharness.git
 if test ! -n "$clonedir" ; then
   clonedir=lib
 fi
 sharnessdir=sharness
 
-if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version"
+if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_${version}_p${patch_version}"
 then
   # There is the right version file. Great, we are done!
   exit 0
@@ -24,11 +26,20 @@ die() {
   exit 1
 }
 
+apply_patches() {
+  git config --local user.email "noone@nowhere"
+  git config --local user.name "No One"
+  git am ../0001-Generate-partial-JUnit-reports.patch
+
+  touch "SHARNESS_VERSION_${version}_p${patch_version}" || die "Could not create 'SHARNESS_VERSION_${version}_p${patch_version}'"
+}
+
 checkout_version() {
   git checkout "$version" || die "Could not checkout '$version'"
   rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'"
-  touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'"
   echo "Sharness version $version is checked out!"
+
+  apply_patches
 }
 
 if test -d "$clonedir/$sharnessdir/.git"

test/sharness/lib/test-lib.sh

@@ -214,6 +214,7 @@ test_launch_ipfs_daemon() {
 
   test_expect_success "'ipfs daemon' succeeds" '
     ipfs daemon $args >actual_daemon 2>daemon_err &
+    IPFS_PID=$!
   '
 
   # wait for api file to show up
@@ -225,7 +226,6 @@ test_launch_ipfs_daemon() {
 
   # we say the daemon is ready when the API server is ready.
   test_expect_success "'ipfs daemon' is ready" '
-    IPFS_PID=$! &&
     pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
     test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout
   '

test/sharness/t0181-private-network.sh

@@ -29,7 +29,7 @@ pnet_key() {
   random 16
 }
 
-pnet_key > $IPFS_PATH/swarm.key
+pnet_key > "${IPFS_PATH}/swarm.key"
 
 LIBP2P_FORCE_PNET=1 test_launch_ipfs_daemon
 
@@ -42,7 +42,7 @@ set_key() {
   node="$1"
   keyfile="$2"
 
-  cp "$keyfile" "$IPTB_ROOT/$node/swarm.key"
+  cp "$keyfile" "${IPTB_ROOT}/${node}/swarm.key"
 }
 
 pnet_key > key1