kubo/test/integration/three_legged_cat_test.go
Marten Seemann 514411bedb
feat: opt-in Swarm.ResourceMgr (go-libp2p v0.18) (#8680)
* update go-libp2p to v0.18.0

* initialize the resource manager

* add resource manager stats/limit commands

* load limit file when building resource manager

* log absent limit file

* write rcmgr to file when IPFS_DEBUG_RCMGR is set

* fix: mark swarm limit|stats as experimental

* feat(cfg): opt-in Swarm.ResourceMgr

This ensures we can safely test the resource manager without impacting
default behavior.

- Resource manager is disabled by default
    - Default for Swarm.ResourceMgr.Enabled is false for now
- Swarm.ResourceMgr.Limits allows user to tweak limits per specific
  scope in a way that is persisted across restarts
- 'ipfs swarm limit system' outputs human-readable json
- 'ipfs swarm limit system new-limits.json' sets new runtime limits
  (but does not change Swarm.ResourceMgr.Limits in the config)

Conventions to make libp2p devs life easier:
- 'IPFS_RCMGR=1 ipfs daemon' overrides the config and enables resource manager
- 'limit.json' overrides implicit defaults from libp2p (if present)

* docs(config): small tweaks

* fix: skip libp2p.ResourceManager if disabled

This ensures 'ipfs swarm limit|stats' work only when enabled.

* fix: use NullResourceManager when disabled

This reverts commit b19f7c9eca.
after clarification feedback from
https://github.com/ipfs/go-ipfs/pull/8680#discussion_r841680182

* style: rename IPFS_RCMGR to LIBP2P_RCMGR

preexisting libp2p toggles use LIBP2P_ prefix

* test: Swarm.ResourceMgr

* fix: location of opt-in limit.json and rcmgr.json.gz

Places these files inside of IPFS_PATH

* Update docs/config.md

* feat: expose rcmgr metrics when enabled (#8785)

* add metrics for the resource manager
* export protocol and service name in Prometheus metrics
* fix: expose rcmgr metrics only when enabled

Co-authored-by: Marcin Rataj <lidel@lidel.org>

* refactor: rcmgr_metrics.go

* refactor: rcmgr_defaults.go

This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled

We keep vendored copy to ensure go-ipfs is not impacted when go-libp2p
decides to change defaults in any of the future releases.

* refactor: adjustedDefaultLimits

Cleans up the way we initialize defaults and adds a fix for case
when connection manager runs with high limits.

It also hides `Swarm.ResourceMgr.Limits` until we have a better
understanding what syntax makes sense.

* chore: cleanup after a review

* fix: restore go-ipld-prime v0.14.2

* fix: restore go-ds-flatfs v0.5.1

Co-authored-by: Lucas Molas <schomatis@gmail.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>
2022-04-07 21:06:35 -04:00

141 lines
3.2 KiB
Go

package integrationtest
import (
"bytes"
"context"
"errors"
"io"
"math"
"testing"
"time"
bootstrap2 "github.com/ipfs/go-ipfs/core/bootstrap"
"github.com/ipfs/go-ipfs/core/coreapi"
mock "github.com/ipfs/go-ipfs/core/mock"
"github.com/ipfs/go-ipfs/thirdparty/unit"
files "github.com/ipfs/go-ipfs-files"
"github.com/libp2p/go-libp2p-core/peer"
testutil "github.com/libp2p/go-libp2p-testing/net"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
)
func TestThreeLeggedCatTransfer(t *testing.T) {
conf := testutil.LatencyConfig{
NetworkLatency: 0,
RoutingLatency: 0,
BlockstoreLatency: 0,
}
if err := RunThreeLeggedCat(RandomBytes(100*unit.MB), conf); err != nil {
t.Fatal(err)
}
}
func TestThreeLeggedCatDegenerateSlowBlockstore(t *testing.T) {
SkipUnlessEpic(t)
conf := testutil.LatencyConfig{BlockstoreLatency: 50 * time.Millisecond}
if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
t.Fatal(err)
}
}
func TestThreeLeggedCatDegenerateSlowNetwork(t *testing.T) {
SkipUnlessEpic(t)
conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond}
if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
t.Fatal(err)
}
}
func TestThreeLeggedCatDegenerateSlowRouting(t *testing.T) {
SkipUnlessEpic(t)
conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond}
if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
t.Fatal(err)
}
}
func TestThreeLeggedCat100MBMacbookCoastToCoast(t *testing.T) {
SkipUnlessEpic(t)
conf := testutil.LatencyConfig{}.NetworkNYtoSF().BlockstoreSlowSSD2014().RoutingSlow()
if err := RunThreeLeggedCat(RandomBytes(100*unit.MB), conf); err != nil {
t.Fatal(err)
}
}
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create network
mn := mocknet.New()
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
bootstrap, err := mock.MockPublicNode(ctx, mn)
if err != nil {
return err
}
defer bootstrap.Close()
adder, err := mock.MockPublicNode(ctx, mn)
if err != nil {
return err
}
defer adder.Close()
catter, err := mock.MockPublicNode(ctx, mn)
if err != nil {
return err
}
defer catter.Close()
adderApi, err := coreapi.NewCoreAPI(adder)
if err != nil {
return err
}
catterApi, err := coreapi.NewCoreAPI(catter)
if err != nil {
return err
}
err = mn.LinkAll()
if err != nil {
return err
}
bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID())
bcfg := bootstrap2.BootstrapConfigWithPeers([]peer.AddrInfo{bis})
if err := adder.Bootstrap(bcfg); err != nil {
return err
}
if err := catter.Bootstrap(bcfg); err != nil {
return err
}
added, err := adderApi.Unixfs().Add(ctx, files.NewBytesFile(data))
if err != nil {
return err
}
readerCatted, err := catterApi.Unixfs().Get(ctx, added)
if err != nil {
return err
}
// verify
var bufout bytes.Buffer
_, err = io.Copy(&bufout, readerCatted.(io.Reader))
if err != nil {
return err
}
if !bytes.Equal(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}