Merge branch 'master' into merge-release-v0.19.2

This commit is contained in:
Henrique Dias 2023-05-03 12:57:21 +02:00
parent d1b0083d60
commit 5f5c16e2cc
No known key found for this signature in database
8 changed files with 261 additions and 315 deletions

3
.github/CODEOWNERS vendored
View File

@ -2,6 +2,9 @@
# request that modifies code that they own. Code owners are not automatically
# requested to review draft pull requests.
# Default
* @ipfs/kubo-maintainers
# HTTP Gateway
core/corehttp/ @lidel
test/sharness/*gateway*.sh @lidel

View File

@ -0,0 +1,116 @@
package testutils
import (
"fmt"
"io"
"math/rand"
"os"
"path"
"time"
)
var AlphabetEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_")
var AlphabetHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ")
type RandFiles struct {
Rand *rand.Rand
FileSize int // the size per file.
FilenameSize int
Alphabet []rune // for filenames
FanoutDepth int // how deep the hierarchy goes
FanoutFiles int // how many files per dir
FanoutDirs int // how many dirs per dir
RandomSize bool // randomize file sizes
RandomFanout bool // randomize fanout numbers
}
func NewRandFiles() *RandFiles {
return &RandFiles{
Rand: rand.New(rand.NewSource(time.Now().UnixNano())),
FileSize: 4096,
FilenameSize: 16,
Alphabet: AlphabetEasy,
FanoutDepth: 2,
FanoutDirs: 5,
FanoutFiles: 10,
RandomSize: true,
}
}
func (r *RandFiles) WriteRandomFiles(root string, depth int) error {
numfiles := r.FanoutFiles
if r.RandomFanout {
numfiles = rand.Intn(r.FanoutFiles) + 1
}
for i := 0; i < numfiles; i++ {
if err := r.WriteRandomFile(root); err != nil {
return err
}
}
if depth+1 <= r.FanoutDepth {
numdirs := r.FanoutDirs
if r.RandomFanout {
numdirs = r.Rand.Intn(numdirs) + 1
}
for i := 0; i < numdirs; i++ {
if err := r.WriteRandomDir(root, depth+1); err != nil {
return err
}
}
}
return nil
}
func (r *RandFiles) RandomFilename(length int) string {
b := make([]rune, length)
for i := range b {
b[i] = r.Alphabet[r.Rand.Intn(len(r.Alphabet))]
}
return string(b)
}
func (r *RandFiles) WriteRandomFile(root string) error {
filesize := int64(r.FileSize)
if r.RandomSize {
filesize = r.Rand.Int63n(filesize) + 1
}
n := rand.Intn(r.FilenameSize-4) + 4
name := r.RandomFilename(n)
filepath := path.Join(root, name)
f, err := os.Create(filepath)
if err != nil {
return fmt.Errorf("creating random file: %w", err)
}
if _, err := io.CopyN(f, r.Rand, filesize); err != nil {
return fmt.Errorf("copying random file: %w", err)
}
return f.Close()
}
func (r *RandFiles) WriteRandomDir(root string, depth int) error {
if depth > r.FanoutDepth {
return nil
}
n := rand.Intn(r.FilenameSize-4) + 4
name := r.RandomFilename(n)
root = path.Join(root, name)
if err := os.MkdirAll(root, 0755); err != nil {
return fmt.Errorf("creating random dir: %w", err)
}
err := r.WriteRandomFiles(root, depth)
if err != nil {
return fmt.Errorf("writing random files in random dir: %w", err)
}
return nil
}

127
test/cli/transports_test.go Normal file
View File

@ -0,0 +1,127 @@
package cli
import (
"os"
"path/filepath"
"testing"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/ipfs/kubo/test/cli/testutils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTransports(t *testing.T) {
disableRouting := func(nodes harness.Nodes) {
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Routing.Type = config.NewOptionalString("none")
cfg.Bootstrap = nil
})
})
}
checkSingleFile := func(nodes harness.Nodes) {
s := testutils.RandomStr(100)
hash := nodes[0].IPFSAddStr(s)
nodes.ForEachPar(func(n *harness.Node) {
val := n.IPFS("cat", hash).Stdout.String()
assert.Equal(t, s, val)
})
}
checkRandomDir := func(nodes harness.Nodes) {
randDir := filepath.Join(nodes[0].Dir, "foobar")
require.NoError(t, os.Mkdir(randDir, 0777))
rf := testutils.NewRandFiles()
rf.FanoutDirs = 3
rf.FanoutFiles = 6
require.NoError(t, rf.WriteRandomFiles(randDir, 4))
hash := nodes[1].IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
nodes.ForEachPar(func(n *harness.Node) {
res := n.RunIPFS("refs", "-r", hash)
assert.Equal(t, 0, res.ExitCode())
})
}
runTests := func(nodes harness.Nodes) {
checkSingleFile(nodes)
checkRandomDir(nodes)
}
tcpNodes := func(t *testing.T) harness.Nodes {
nodes := harness.NewT(t).NewNodes(2).Init()
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/0"}
cfg.Swarm.Transports.Network.QUIC = config.False
cfg.Swarm.Transports.Network.Relay = config.False
cfg.Swarm.Transports.Network.WebTransport = config.False
cfg.Swarm.Transports.Network.Websocket = config.False
})
})
disableRouting(nodes)
return nodes
}
t.Run("tcp", func(t *testing.T) {
t.Parallel()
nodes := tcpNodes(t).StartDaemons().Connect()
runTests(nodes)
})
t.Run("tcp with mplex", func(t *testing.T) {
t.Parallel()
nodes := tcpNodes(t)
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Swarm.Transports.Multiplexers.Yamux = config.Disabled
})
})
nodes.StartDaemons().Connect()
runTests(nodes)
})
t.Run("tcp with NOISE", func(t *testing.T) {
t.Parallel()
nodes := tcpNodes(t)
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Swarm.Transports.Security.TLS = config.Disabled
})
})
nodes.StartDaemons().Connect()
runTests(nodes)
})
t.Run("QUIC", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(5).Init()
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1"}
cfg.Swarm.Transports.Network.QUIC = config.True
cfg.Swarm.Transports.Network.TCP = config.False
})
})
disableRouting(nodes)
nodes.StartDaemons().Connect()
runTests(nodes)
})
t.Run("QUIC", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(5).Init()
nodes.ForEachPar(func(n *harness.Node) {
n.UpdateConfig(func(cfg *config.Config) {
cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1/webtransport"}
cfg.Swarm.Transports.Network.QUIC = config.True
cfg.Swarm.Transports.Network.WebTransport = config.True
})
})
disableRouting(nodes)
nodes.StartDaemons().Connect()
runTests(nodes)
})
}

View File

@ -50,19 +50,19 @@ test_path_cmp() {
# Docker
# This takes a Dockerfile, and a build context directory
# This takes a Dockerfile, a tag name, and a build context directory
docker_build() {
docker build --rm -f "$1" "$2" | ansi_strip
docker build --rm --tag "$1" --file "$2" "$3" | ansi_strip
}
# This takes an image as argument and writes a docker ID on stdout
docker_run() {
docker run -d "$1"
docker run --detach "$1"
}
# This takes a docker ID and a command as arguments
docker_exec() {
docker exec -t "$1" /bin/sh -c "$2"
docker exec --tty "$1" /bin/sh -c "$2"
}
# This takes a docker ID as argument
@ -72,12 +72,12 @@ docker_stop() {
# This takes a docker ID as argument
docker_rm() {
docker rm -f -v "$1" > /dev/null
docker rm --force --volumes "$1" > /dev/null
}
# This takes a docker image name as argument
docker_rmi() {
docker rmi -f "$1" > /dev/null
docker rmi --force "$1" > /dev/null
}
# Test whether all the expected lines are included in a file. The file

View File

@ -27,18 +27,12 @@ TEST_TRASH_DIR=$(pwd)
TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR")
TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR")
APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR")
IMAGE_TAG=kubo_test
test_expect_success "docker image build succeeds" '
docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" | tee build-actual ||
docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" ||
test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" ||
test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ||
test_fsh cat build-actual
'
test_expect_success "docker image build output looks good" '
SUCCESS_LINE=$(egrep "^Successfully built" build-actual) &&
IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") ||
test_fsh cat build-actual
test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR"
'
test_expect_success "write init scripts" '
@ -52,7 +46,7 @@ test_expect_success "docker image runs" '
-p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 \
-v "$PWD/001.sh":/container-init.d/001.sh \
-v "$PWD/002.sh":/container-init.d/002.sh \
"$IMAGE_ID")
"$IMAGE_TAG")
'
test_expect_success "docker container gateway is up" '
@ -100,5 +94,5 @@ test_expect_success "stop docker container" '
'
docker_rm "$DOC_ID"
docker_rmi "$IMAGE_ID"
docker_rmi "$IMAGE_TAG"
test_done

View File

@ -24,10 +24,10 @@ TEST_TRASH_DIR=$(pwd)
TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR")
TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR")
APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR")
IMAGE_TAG=kubo_migrate
test_expect_success "docker image build succeeds" '
docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual &&
IMAGE_ID=$(tail -n1 actual | cut -d " " -f 3)
docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR"
'
test_init_ipfs
@ -53,7 +53,7 @@ test_expect_success "startup fake dists server" '
'
test_expect_success "docker image runs" '
DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID")
DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG")
'
test_expect_success "docker container tries to pull migrations from netcat" '
@ -78,6 +78,5 @@ test_expect_success "correct version was requested" '
'
docker_rm "$DOC_ID"
docker_rmi "$IMAGE_ID"
docker_rmi "$IMAGE_TAG"
test_done

View File

@ -1,178 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2017 Jeromy Johnson
# MIT Licensed; see the LICENSE file in this repository.
#
test_description="Test two ipfs nodes transferring a file"
. lib/test-lib.sh
check_file_fetch() {
node=$1
fhash=$2
fname=$3
test_expect_success "can fetch file" '
ipfsi $node cat $fhash > fetch_out
'
test_expect_success "file looks good" '
test_cmp $fname fetch_out
'
}
check_dir_fetch() {
node=$1
ref=$2
test_expect_success "node can fetch all refs for dir" '
ipfsi $node refs -r $ref > /dev/null
'
}
run_single_file_test() {
test_expect_success "add a file on node1" '
random 1000000 > filea &&
FILEA_HASH=$(ipfsi 1 add -q filea)
'
check_file_fetch 0 $FILEA_HASH filea
}
run_random_dir_test() {
test_expect_success "create a bunch of random files" '
random-files -depth=3 -dirs=4 -files=5 -seed=5 foobar > /dev/null
'
test_expect_success "add those on node 0" '
DIR_HASH=$(ipfsi 0 add -r -Q foobar)
'
check_dir_fetch 1 $DIR_HASH
}
flaky_advanced_test() {
startup_cluster 2 "$@"
test_expect_success "clean repo before test" '
ipfsi 0 repo gc > /dev/null &&
ipfsi 1 repo gc > /dev/null
'
run_single_file_test
run_random_dir_test
test_expect_success "gather bitswap stats" '
ipfsi 0 bitswap stat -v > stat0 &&
ipfsi 1 bitswap stat -v > stat1
'
test_expect_success "shut down nodes" '
iptb stop && iptb_wait_stop
'
# NOTE: data transferred stats checks are flaky
# trying to debug them by printing out the stats hides the flakiness
# my theory is that the extra time cat calls take to print out the stats
# allow for proper cleanup to happen
go-sleep 1s
}
run_advanced_test() {
# TODO: investigate why flaky_advanced_test is flaky
# Context: https://github.com/ipfs/kubo/pull/9486
# sometimes, bitswap status returns unexpected block transfers
# and everyone has been re-running circleci until is passes for at least a year.
# this re-runs test until it passes or a timeout hits
BLOCKS_0=126
BLOCKS_1=5
DATA_0=228113
DATA_1=1000256
for i in $(test_seq 1 600); do
flaky_advanced_test
(grep -q "$DATA_0" stat0 && grep -q "$DATA_1" stat1) && break
go-sleep 100ms
done
test_expect_success "node0 data transferred looks correct" '
test_should_contain "blocks sent: $BLOCKS_0" stat0 &&
test_should_contain "blocks received: $BLOCKS_1" stat0 &&
test_should_contain "data sent: $DATA_0" stat0 &&
test_should_contain "data received: $DATA_1" stat0
'
test_expect_success "node1 data transferred looks correct" '
test_should_contain "blocks received: $BLOCKS_0" stat1 &&
test_should_contain "blocks sent: $BLOCKS_1" stat1 &&
test_should_contain "data received: $DATA_0" stat1 &&
test_should_contain "data sent: $DATA_1" stat1
'
}
test_expect_success "set up tcp testbed" '
iptb testbed create -type localipfs -count 2 -force -init
'
test_expect_success "disable routing, use direct peering" '
iptb run -- ipfs config Routing.Type none &&
iptb run -- ipfs config --json Bootstrap "[]"
'
# Test TCP transport
echo "Testing TCP"
addrs='"[\"/ip4/127.0.0.1/tcp/0\"]"'
test_expect_success "use TCP only" '
iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
iptb run -- ipfs config --json Swarm.Transports.Network.QUIC false &&
iptb run -- ipfs config --json Swarm.Transports.Network.Relay false &&
iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport false &&
iptb run -- ipfs config --json Swarm.Transports.Network.Websocket false
'
run_advanced_test
# test multiplex muxer
echo "Running TCP tests with mplex"
test_expect_success "disable yamux" '
iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
'
run_advanced_test
test_expect_success "re-enable yamux" '
iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux null
'
# test Noise
echo "Running TCP tests with NOISE"
test_expect_success "use noise only" '
iptb run -- ipfs config --json Swarm.Transports.Security.TLS false
'
run_advanced_test
test_expect_success "re-enable TLS" '
iptb run -- ipfs config --json Swarm.Transports.Security.TLS null
'
# test QUIC
echo "Running advanced tests over QUIC"
addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"'
test_expect_success "use QUIC only" '
iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true &&
iptb run -- ipfs config --json Swarm.Transports.Network.TCP false
'
run_advanced_test
# test WebTransport
echo "Running advanced tests over WebTransport"
addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"'
test_expect_success "use WebTransport only" '
iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true &&
iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport true
'
run_advanced_test
test_done

View File

@ -1,115 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2015 Jeromy Johnson
# MIT Licensed; see the LICENSE file in this repository.
#
test_description="Test multiple ipfs nodes"
. lib/test-lib.sh
check_file_fetch() {
node=$1
fhash=$2
fname=$3
test_expect_success "can fetch file" '
ipfsi $node cat $fhash > fetch_out
'
test_expect_success "file looks good" '
test_cmp $fname fetch_out
'
}
check_dir_fetch() {
node=$1
ref=$2
test_expect_success "node can fetch all refs for dir" '
ipfsi $node refs -r $ref > /dev/null
'
}
run_single_file_test() {
test_expect_success "add a file on node1" '
random 1000000 > filea &&
FILEA_HASH=$(ipfsi 1 add -q filea)
'
check_file_fetch 4 $FILEA_HASH filea
check_file_fetch 3 $FILEA_HASH filea
check_file_fetch 2 $FILEA_HASH filea
check_file_fetch 1 $FILEA_HASH filea
check_file_fetch 0 $FILEA_HASH filea
}
run_random_dir_test() {
test_expect_success "create a bunch of random files" '
random-files -depth=4 -dirs=3 -files=6 foobar > /dev/null
'
test_expect_success "add those on node 2" '
DIR_HASH=$(ipfsi 2 add -r -Q foobar)
'
check_dir_fetch 0 $DIR_HASH
check_dir_fetch 1 $DIR_HASH
check_dir_fetch 2 $DIR_HASH
check_dir_fetch 3 $DIR_HASH
check_dir_fetch 4 $DIR_HASH
}
run_basic_test() {
startup_cluster 5
run_single_file_test
test_expect_success "shut down nodes" '
iptb stop && iptb_wait_stop
'
}
run_advanced_test() {
startup_cluster 5 "$@"
run_single_file_test
run_random_dir_test
test_expect_success "shut down nodes" '
iptb stop && iptb_wait_stop ||
test_fsh tail -n +1 .iptb/testbeds/default/*/daemon.std*
'
}
test_expect_success "set up /tcp testbed" '
iptb testbed create -type localipfs -count 5 -force -init
'
# test default configuration
run_advanced_test
# test multiplex muxer
test_expect_success "disable yamux" '
iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
'
run_advanced_test
test_expect_success "set up /ws testbed" '
iptb testbed create -type localipfs -count 5 -attr listentype,ws -force -init
'
# test default configuration
run_advanced_test
# test multiplex muxer
test_expect_success "disable yamux" '
iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
'
run_advanced_test
test_done