feat: re-enable docker sharness tests (#8808)

The Docker sharness tests were disabled years ago when go-ipfs moved
from Travis to CircleCI. This makes the tweaks necessary to re-enable
them.

The Docker image has since moved to be based on BusyBox which doesn't
have the requisite wget version for the existing tests to work, so
this adds some functionality to the pollEndpoint program to support
polling HTTP endpoints as well.

.circleci/main.yml

@@ -154,7 +154,7 @@ jobs:
     working_directory: ~/ipfs/go-ipfs
     environment:
         <<: *default_environment
-        TEST_NO_DOCKER: 1
+        TEST_NO_DOCKER: 0
         TEST_NO_FUSE: 1
         TEST_VERBOSE: 1
     steps:

.dockerignore

@@ -5,6 +5,8 @@ Dockerfile.fast
 !.git/refs/
 !.git/packed-refs
 test/sharness/lib/sharness/
+test/sharness/trash*
+rb-pinning-service-api/
 
 # The Docker client might not be running on Linux
 # so delete any compiled binaries

test/dependencies/pollEndpoint/main.go

@@ -2,7 +2,11 @@
 package main
 
 import (
+	"context"
 	"flag"
+	"io"
+	"net"
+	"net/http"
 	"os"
 	"time"
 
@@ -15,6 +19,8 @@ var (
 	host    = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
 	tries   = flag.Int("tries", 10, "how many tries to make before failing")
 	timeout = flag.Duration("tout", time.Second, "how long to wait between attempts")
+	httpURL = flag.String("http-url", "", "HTTP URL to fetch")
+	httpOut = flag.Bool("http-out", false, "Print the HTTP response body to stdout")
 	verbose = flag.Bool("v", false, "verbose logging")
 )
 
@@ -37,18 +43,76 @@ func main() {
 	start := time.Now()
 	log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr)
 
-	for *tries > 0 {
+	connTries := *tries
+	for connTries > 0 {
 		c, err := manet.Dial(addr)
 		if err == nil {
 			log.Debugf("ok -  endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start))
 			c.Close()
-			os.Exit(0)
+			break
 		}
 		log.Debug("connect failed: ", err)
 		time.Sleep(*timeout)
-		*tries--
+		connTries--
 	}
 
-	log.Error("failed.")
+	if err != nil {
+		goto Fail
+	}
+
+	if *httpURL != "" {
+		dialer := &connDialer{addr: addr}
+		httpClient := http.Client{Transport: &http.Transport{
+			DialContext: dialer.DialContext,
+		}}
+		reqTries := *tries
+		for reqTries > 0 {
+			try := (*tries - reqTries) + 1
+			log.Debugf("trying HTTP req %d: '%s'", try, *httpURL)
+			if tryHTTPGet(&httpClient, *httpURL) {
+				log.Debugf("HTTP req %d to '%s' succeeded", try, *httpURL)
+				goto Success
+			}
+			log.Debugf("HTTP req %d to '%s' failed", try, *httpURL)
+			time.Sleep(*timeout)
+			reqTries--
+		}
+		goto Fail
+	}
+
+Success:
+	os.Exit(0)
+
+Fail:
+	log.Error("failed")
 	os.Exit(1)
 }
+
+func tryHTTPGet(client *http.Client, url string) bool {
+	resp, err := client.Get(*httpURL)
+	if resp != nil && resp.Body != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		return false
+	}
+	if resp.StatusCode != http.StatusOK {
+		return false
+	}
+	if *httpOut {
+		_, err := io.Copy(os.Stdout, resp.Body)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return true
+}
+
+type connDialer struct {
+	addr ma.Multiaddr
+}
+
+func (d connDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	return (&manet.Dialer{}).DialContext(ctx, d.addr)
+}

test/ipfs-test-lib.sh

@@ -62,12 +62,7 @@ docker_run() {
 
 # This takes a docker ID and a command as arguments
 docker_exec() {
-    if test "$CIRCLE" = 1
+    docker exec -t "$1" /bin/sh -c "$2"
-    then
-        sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $1)" -- /bin/bash -c "$2"
-    else
-	docker exec -t "$1" /bin/bash -c "$2"
-    fi
 }
 
 # This takes a docker ID as argument

test/sharness/t0300-docker-image.sh

@@ -29,30 +29,28 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR")
 APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR")
 
 test_expect_success "docker image build succeeds" '
-  docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual ||
+  docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >build-actual ||
   test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" ||
   test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ||
-  test_fsh cat actual
+  test_fsh cat build-actual
 '
 
 test_expect_success "docker image build output looks good" '
-  SUCCESS_LINE=$(egrep "^Successfully built" actual) &&
+  SUCCESS_LINE=$(egrep "^Successfully built" build-actual) &&
   IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") ||
-  test_fsh cat actual
+  test_fsh cat build-actual
 '
 
 test_expect_success "docker image runs" '
-  DOC_ID=$(docker_run "$IMAGE_ID")
+  DOC_ID=$(docker run -d -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 "$IMAGE_ID")
 '
 
-test_expect_success "docker image gateway is up" '
+test_expect_success "docker container gateway is up" '
-  docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/8080 -http-url http://localhost:8080/api/v0/version -v -tries 30 -tout 1s
-    -q -O - http://localhost:8080/version >/dev/null"
 '
 
-test_expect_success "docker image API is up" '
+test_expect_success "docker container API is up" '
-  docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -v -tries 30 -tout 1s
-    -q -O - http://localhost:5001/api/v0/version >/dev/null"
 '
 
 test_expect_success "simple ipfs add/cat can be run in docker container" '
@@ -63,8 +61,7 @@ test_expect_success "simple ipfs add/cat can be run in docker container" '
 '
 
 read testcode <<EOF
-  docker exec -i "$DOC_ID" wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -http-out | grep Commit | cut -d" " -f2 >actual ; \
-    -q -O - http://localhost:8080/version | grep Commit | cut -d" " -f2 >actual ; \
   test -s actual ; \
   docker exec -i "$DOC_ID" ipfs version --enc json \
     | sed 's/^.*"Commit":"\\\([^"]*\\\)".*$/\\\1/g' >expected ; \

test/sharness/t0301-docker-migrate.sh

@@ -32,6 +32,10 @@ test_expect_success "docker image build succeeds" '
 
 test_init_ipfs
 
+test_expect_success "configure migration sources" '
+  ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]"
+'
+
 test_expect_success "make repo be version 4" '
   echo 4 > "$IPFS_PATH/version"
 '
@@ -43,17 +47,13 @@ test_expect_success "setup http response" '
   echo "v1.1.1" >> vers_resp
 '
 
-pretend_server() {
-    socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr 'SYSTEM:cat vers_resp'!!STDERR &
-}
-
 test_expect_success "startup fake dists server" '
-  pretend_server > dist_serv_out &
+  ( socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr "SYSTEM:cat vers_resp"!!STDERR 2> dist_serv_out ) &
   echo $! > netcat_pid
 '
 
 test_expect_success "docker image runs" '
-  DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host -e IPFS_DIST_PATH="http://localhost:17233" "$IMAGE_ID" --migrate)
+  DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID")
 '
 
 test_expect_success "docker container tries to pull migrations from netcat" '
@@ -74,7 +74,7 @@ test_expect_success "kill the net cat" '
 '
 
 test_expect_success "correct version was requested" '
-  grep "/fs-repo-migrations/v1.1.1/fs-repo-migrations_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null
+  grep "/fs-repo-6-to-7/v1.1.1/fs-repo-6-to-7_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null
 '
 
 test_done