diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5151ad178..7d17855e0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,6 +2,9 @@ # request that modifies code that they own. Code owners are not automatically # requested to review draft pull requests. +# Default +* @ipfs/kubo-maintainers + # HTTP Gateway core/corehttp/ @lidel test/sharness/*gateway*.sh @lidel diff --git a/test/cli/testutils/random_files.go b/test/cli/testutils/random_files.go new file mode 100644 index 000000000..0e550a125 --- /dev/null +++ b/test/cli/testutils/random_files.go @@ -0,0 +1,116 @@ +package testutils + +import ( + "fmt" + "io" + "math/rand" + "os" + "path" + "time" +) + +var AlphabetEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_") +var AlphabetHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ") + +type RandFiles struct { + Rand *rand.Rand + FileSize int // the size per file. + FilenameSize int + Alphabet []rune // for filenames + + FanoutDepth int // how deep the hierarchy goes + FanoutFiles int // how many files per dir + FanoutDirs int // how many dirs per dir + + RandomSize bool // randomize file sizes + RandomFanout bool // randomize fanout numbers +} + +func NewRandFiles() *RandFiles { + return &RandFiles{ + Rand: rand.New(rand.NewSource(time.Now().UnixNano())), + FileSize: 4096, + FilenameSize: 16, + Alphabet: AlphabetEasy, + FanoutDepth: 2, + FanoutDirs: 5, + FanoutFiles: 10, + RandomSize: true, + } +} + +func (r *RandFiles) WriteRandomFiles(root string, depth int) error { + numfiles := r.FanoutFiles + if r.RandomFanout { + numfiles = rand.Intn(r.FanoutFiles) + 1 + } + + for i := 0; i < numfiles; i++ { + if err := r.WriteRandomFile(root); err != nil { + return err + } + } + + if depth+1 <= r.FanoutDepth { + numdirs := r.FanoutDirs + if r.RandomFanout { + numdirs = r.Rand.Intn(numdirs) + 1 + } + + for i := 0; i < numdirs; i++ { + if err := r.WriteRandomDir(root, depth+1); err != nil { + return err + } + } + } + + return nil +} + +func (r *RandFiles) RandomFilename(length int) string { + b := make([]rune, length) + for i := range b { + b[i] = r.Alphabet[r.Rand.Intn(len(r.Alphabet))] + } + return string(b) +} + +func (r *RandFiles) WriteRandomFile(root string) error { + filesize := int64(r.FileSize) + if r.RandomSize { + filesize = r.Rand.Int63n(filesize) + 1 + } + + n := rand.Intn(r.FilenameSize-4) + 4 + name := r.RandomFilename(n) + filepath := path.Join(root, name) + f, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("creating random file: %w", err) + } + + if _, err := io.CopyN(f, r.Rand, filesize); err != nil { + return fmt.Errorf("copying random file: %w", err) + } + + return f.Close() +} + +func (r *RandFiles) WriteRandomDir(root string, depth int) error { + if depth > r.FanoutDepth { + return nil + } + + n := rand.Intn(r.FilenameSize-4) + 4 + name := r.RandomFilename(n) + root = path.Join(root, name) + if err := os.MkdirAll(root, 0755); err != nil { + return fmt.Errorf("creating random dir: %w", err) + } + + err := r.WriteRandomFiles(root, depth) + if err != nil { + return fmt.Errorf("writing random files in random dir: %w", err) + } + return nil +} diff --git a/test/cli/transports_test.go b/test/cli/transports_test.go new file mode 100644 index 000000000..78f21fd05 --- /dev/null +++ b/test/cli/transports_test.go @@ -0,0 +1,127 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/ipfs/kubo/test/cli/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTransports(t *testing.T) { + disableRouting := func(nodes harness.Nodes) { + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Routing.Type = config.NewOptionalString("none") + cfg.Bootstrap = nil + }) + }) + } + checkSingleFile := func(nodes harness.Nodes) { + s := testutils.RandomStr(100) + hash := nodes[0].IPFSAddStr(s) + nodes.ForEachPar(func(n *harness.Node) { + val := n.IPFS("cat", hash).Stdout.String() + assert.Equal(t, s, val) + }) + } + checkRandomDir := func(nodes harness.Nodes) { + randDir := filepath.Join(nodes[0].Dir, "foobar") + require.NoError(t, os.Mkdir(randDir, 0777)) + rf := testutils.NewRandFiles() + rf.FanoutDirs = 3 + rf.FanoutFiles = 6 + require.NoError(t, rf.WriteRandomFiles(randDir, 4)) + + hash := nodes[1].IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + nodes.ForEachPar(func(n *harness.Node) { + res := n.RunIPFS("refs", "-r", hash) + assert.Equal(t, 0, res.ExitCode()) + }) + } + + runTests := func(nodes harness.Nodes) { + checkSingleFile(nodes) + checkRandomDir(nodes) + } + + tcpNodes := func(t *testing.T) harness.Nodes { + nodes := harness.NewT(t).NewNodes(2).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/0"} + cfg.Swarm.Transports.Network.QUIC = config.False + cfg.Swarm.Transports.Network.Relay = config.False + cfg.Swarm.Transports.Network.WebTransport = config.False + cfg.Swarm.Transports.Network.Websocket = config.False + }) + }) + disableRouting(nodes) + return nodes + } + + t.Run("tcp", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t).StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("tcp with mplex", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t) + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Swarm.Transports.Multiplexers.Yamux = config.Disabled + }) + }) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("tcp with NOISE", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t) + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Swarm.Transports.Security.TLS = config.Disabled + }) + }) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("QUIC", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1"} + cfg.Swarm.Transports.Network.QUIC = config.True + cfg.Swarm.Transports.Network.TCP = config.False + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("QUIC", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1/webtransport"} + cfg.Swarm.Transports.Network.QUIC = config.True + cfg.Swarm.Transports.Network.WebTransport = config.True + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + +} diff --git a/test/ipfs-test-lib.sh b/test/ipfs-test-lib.sh index eabf17020..a3bc32bbb 100644 --- a/test/ipfs-test-lib.sh +++ b/test/ipfs-test-lib.sh @@ -50,19 +50,19 @@ test_path_cmp() { # Docker -# This takes a Dockerfile, and a build context directory +# This takes a Dockerfile, a tag name, and a build context directory docker_build() { - docker build --rm -f "$1" "$2" | ansi_strip + docker build --rm --tag "$1" --file "$2" "$3" | ansi_strip } # This takes an image as argument and writes a docker ID on stdout docker_run() { - docker run -d "$1" + docker run --detach "$1" } # This takes a docker ID and a command as arguments docker_exec() { - docker exec -t "$1" /bin/sh -c "$2" + docker exec --tty "$1" /bin/sh -c "$2" } # This takes a docker ID as argument @@ -72,12 +72,12 @@ docker_stop() { # This takes a docker ID as argument docker_rm() { - docker rm -f -v "$1" > /dev/null + docker rm --force --volumes "$1" > /dev/null } # This takes a docker image name as argument docker_rmi() { - docker rmi -f "$1" > /dev/null + docker rmi --force "$1" > /dev/null } # Test whether all the expected lines are included in a file. The file diff --git a/test/sharness/t0300-docker-image.sh b/test/sharness/t0002-docker-image.sh similarity index 87% rename from test/sharness/t0300-docker-image.sh rename to test/sharness/t0002-docker-image.sh index 3c390a453..11ccf01b7 100755 --- a/test/sharness/t0300-docker-image.sh +++ b/test/sharness/t0002-docker-image.sh @@ -27,18 +27,12 @@ TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") +IMAGE_TAG=kubo_test test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" | tee build-actual || + docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" || test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" || - test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" || - test_fsh cat build-actual -' - -test_expect_success "docker image build output looks good" ' - SUCCESS_LINE=$(egrep "^Successfully built" build-actual) && - IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") || - test_fsh cat build-actual + test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ' test_expect_success "write init scripts" ' @@ -52,7 +46,7 @@ test_expect_success "docker image runs" ' -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 \ -v "$PWD/001.sh":/container-init.d/001.sh \ -v "$PWD/002.sh":/container-init.d/002.sh \ - "$IMAGE_ID") + "$IMAGE_TAG") ' test_expect_success "docker container gateway is up" ' @@ -100,5 +94,5 @@ test_expect_success "stop docker container" ' ' docker_rm "$DOC_ID" -docker_rmi "$IMAGE_ID" +docker_rmi "$IMAGE_TAG" test_done diff --git a/test/sharness/t0301-docker-migrate.sh b/test/sharness/t0003-docker-migrate.sh similarity index 92% rename from test/sharness/t0301-docker-migrate.sh rename to test/sharness/t0003-docker-migrate.sh index 3f7d32f21..ac3c7aee2 100755 --- a/test/sharness/t0301-docker-migrate.sh +++ b/test/sharness/t0003-docker-migrate.sh @@ -24,10 +24,10 @@ TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") +IMAGE_TAG=kubo_migrate test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual && - IMAGE_ID=$(tail -n1 actual | cut -d " " -f 3) + docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" ' test_init_ipfs @@ -53,7 +53,7 @@ test_expect_success "startup fake dists server" ' ' test_expect_success "docker image runs" ' - DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID") + DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG") ' test_expect_success "docker container tries to pull migrations from netcat" ' @@ -78,6 +78,5 @@ test_expect_success "correct version was requested" ' ' docker_rm "$DOC_ID" -docker_rmi "$IMAGE_ID" +docker_rmi "$IMAGE_TAG" test_done - diff --git a/test/sharness/t0125-twonode.sh b/test/sharness/t0125-twonode.sh deleted file mode 100755 index 8ea1c9e5d..000000000 --- a/test/sharness/t0125-twonode.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017 Jeromy Johnson -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test two ipfs nodes transferring a file" - -. lib/test-lib.sh - -check_file_fetch() { - node=$1 - fhash=$2 - fname=$3 - - test_expect_success "can fetch file" ' - ipfsi $node cat $fhash > fetch_out - ' - - test_expect_success "file looks good" ' - test_cmp $fname fetch_out - ' -} - -check_dir_fetch() { - node=$1 - ref=$2 - - test_expect_success "node can fetch all refs for dir" ' - ipfsi $node refs -r $ref > /dev/null - ' -} - -run_single_file_test() { - test_expect_success "add a file on node1" ' - random 1000000 > filea && - FILEA_HASH=$(ipfsi 1 add -q filea) - ' - - check_file_fetch 0 $FILEA_HASH filea -} - -run_random_dir_test() { - test_expect_success "create a bunch of random files" ' - random-files -depth=3 -dirs=4 -files=5 -seed=5 foobar > /dev/null - ' - - test_expect_success "add those on node 0" ' - DIR_HASH=$(ipfsi 0 add -r -Q foobar) - ' - - check_dir_fetch 1 $DIR_HASH -} - -flaky_advanced_test() { - startup_cluster 2 "$@" - - test_expect_success "clean repo before test" ' - ipfsi 0 repo gc > /dev/null && - ipfsi 1 repo gc > /dev/null - ' - - run_single_file_test - - run_random_dir_test - - test_expect_success "gather bitswap stats" ' - ipfsi 0 bitswap stat -v > stat0 && - ipfsi 1 bitswap stat -v > stat1 - ' - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop - ' - - # NOTE: data transferred stats checks are flaky - # trying to debug them by printing out the stats hides the flakiness - # my theory is that the extra time cat calls take to print out the stats - # allow for proper cleanup to happen - go-sleep 1s -} - -run_advanced_test() { - # TODO: investigate why flaky_advanced_test is flaky - # Context: https://github.com/ipfs/kubo/pull/9486 - # sometimes, bitswap status returns unexpected block transfers - # and everyone has been re-running circleci until is passes for at least a year. - # this re-runs test until it passes or a timeout hits - - BLOCKS_0=126 - BLOCKS_1=5 - DATA_0=228113 - DATA_1=1000256 - for i in $(test_seq 1 600); do - flaky_advanced_test - (grep -q "$DATA_0" stat0 && grep -q "$DATA_1" stat1) && break - go-sleep 100ms - done - - test_expect_success "node0 data transferred looks correct" ' - test_should_contain "blocks sent: $BLOCKS_0" stat0 && - test_should_contain "blocks received: $BLOCKS_1" stat0 && - test_should_contain "data sent: $DATA_0" stat0 && - test_should_contain "data received: $DATA_1" stat0 - ' - - test_expect_success "node1 data transferred looks correct" ' - test_should_contain "blocks received: $BLOCKS_0" stat1 && - test_should_contain "blocks sent: $BLOCKS_1" stat1 && - test_should_contain "data received: $DATA_0" stat1 && - test_should_contain "data sent: $DATA_1" stat1 - ' - -} - -test_expect_success "set up tcp testbed" ' - iptb testbed create -type localipfs -count 2 -force -init -' - -test_expect_success "disable routing, use direct peering" ' - iptb run -- ipfs config Routing.Type none && - iptb run -- ipfs config --json Bootstrap "[]" -' - -# Test TCP transport -echo "Testing TCP" -addrs='"[\"/ip4/127.0.0.1/tcp/0\"]"' -test_expect_success "use TCP only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC false && - iptb run -- ipfs config --json Swarm.Transports.Network.Relay false && - iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport false && - iptb run -- ipfs config --json Swarm.Transports.Network.Websocket false -' -run_advanced_test - -# test multiplex muxer -echo "Running TCP tests with mplex" -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' -run_advanced_test - -test_expect_success "re-enable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux null -' -# test Noise -echo "Running TCP tests with NOISE" -test_expect_success "use noise only" ' - iptb run -- ipfs config --json Swarm.Transports.Security.TLS false -' -run_advanced_test - -test_expect_success "re-enable TLS" ' - iptb run -- ipfs config --json Swarm.Transports.Security.TLS null -' - -# test QUIC -echo "Running advanced tests over QUIC" -addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"' -test_expect_success "use QUIC only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true && - iptb run -- ipfs config --json Swarm.Transports.Network.TCP false -' -run_advanced_test - -# test WebTransport -echo "Running advanced tests over WebTransport" -addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"' -test_expect_success "use WebTransport only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true && - iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport true -' -run_advanced_test - -test_done diff --git a/test/sharness/t0130-multinode.sh b/test/sharness/t0130-multinode.sh deleted file mode 100755 index 04746850b..000000000 --- a/test/sharness/t0130-multinode.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2015 Jeromy Johnson -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test multiple ipfs nodes" - -. lib/test-lib.sh - -check_file_fetch() { - node=$1 - fhash=$2 - fname=$3 - - test_expect_success "can fetch file" ' - ipfsi $node cat $fhash > fetch_out - ' - - test_expect_success "file looks good" ' - test_cmp $fname fetch_out - ' -} - -check_dir_fetch() { - node=$1 - ref=$2 - - test_expect_success "node can fetch all refs for dir" ' - ipfsi $node refs -r $ref > /dev/null - ' -} - -run_single_file_test() { - test_expect_success "add a file on node1" ' - random 1000000 > filea && - FILEA_HASH=$(ipfsi 1 add -q filea) - ' - - check_file_fetch 4 $FILEA_HASH filea - check_file_fetch 3 $FILEA_HASH filea - check_file_fetch 2 $FILEA_HASH filea - check_file_fetch 1 $FILEA_HASH filea - check_file_fetch 0 $FILEA_HASH filea -} - -run_random_dir_test() { - test_expect_success "create a bunch of random files" ' - random-files -depth=4 -dirs=3 -files=6 foobar > /dev/null - ' - - test_expect_success "add those on node 2" ' - DIR_HASH=$(ipfsi 2 add -r -Q foobar) - ' - - check_dir_fetch 0 $DIR_HASH - check_dir_fetch 1 $DIR_HASH - check_dir_fetch 2 $DIR_HASH - check_dir_fetch 3 $DIR_HASH - check_dir_fetch 4 $DIR_HASH -} - - -run_basic_test() { - startup_cluster 5 - - run_single_file_test - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop - ' -} - -run_advanced_test() { - startup_cluster 5 "$@" - - run_single_file_test - - run_random_dir_test - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop || - test_fsh tail -n +1 .iptb/testbeds/default/*/daemon.std* - ' -} - -test_expect_success "set up /tcp testbed" ' - iptb testbed create -type localipfs -count 5 -force -init -' - -# test default configuration -run_advanced_test - -# test multiplex muxer -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' -run_advanced_test - -test_expect_success "set up /ws testbed" ' - iptb testbed create -type localipfs -count 5 -attr listentype,ws -force -init -' - -# test default configuration -run_advanced_test - -# test multiplex muxer -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' - -run_advanced_test - - -test_done