core/node: prioritize announcing pin roots, and flat strategy (#10376)

Co-authored-by: Marcin Rataj <lidel@lidel.org>
This commit is contained in:
Henrique Dias 2024-04-09 08:37:23 +02:00 committed by GitHub
parent 24031b9b69
commit 6f2a61e1df
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 201 additions and 222 deletions

View File

@ -129,11 +129,13 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
var keyProvider fx.Option
switch reprovideStrategy {
case "all", "":
keyProvider = fx.Provide(provider.NewBlockstoreProvider)
keyProvider = fx.Provide(newProvidingStrategy(false, false))
case "roots":
keyProvider = fx.Provide(pinnedProviderStrategy(true))
keyProvider = fx.Provide(newProvidingStrategy(true, true))
case "pinned":
keyProvider = fx.Provide(pinnedProviderStrategy(false))
keyProvider = fx.Provide(newProvidingStrategy(true, false))
case "flat":
keyProvider = fx.Provide(provider.NewBlockstoreProvider)
default:
return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
}
@ -149,13 +151,25 @@ func OfflineProviders() fx.Option {
return fx.Provide(provider.NewNoopProvider)
}
func pinnedProviderStrategy(onlyRoots bool) interface{} {
func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
type input struct {
fx.In
Pinner pin.Pinner
Blockstore blockstore.Blockstore
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
}
return func(in input) provider.KeyChanFunc {
return provider.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher)
if onlyRoots {
return provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)
}
if onlyPinned {
return provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher)
}
return provider.NewPrioritizedProvider(
provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
provider.NewBlockstoreProvider(in.Blockstore),
)
}
}

View File

@ -10,6 +10,7 @@
- [Gateway: `/api/v0` is removed](#gateway-apiv0-is-removed)
- [Removed deprecated Object API commands](#removed-deprecated-object-api-commands)
- [No longer publishes loopback and private addresses on DHT](#no-longer-publishes-loopback-and-private-addresses-on-dht)
- [Pin roots are now prioritized when announcing](#pin-roots-are-now-prioritized-when-announcing)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
@ -35,6 +36,10 @@ Kubo no longer keeps track of loopback and private addresses on the LAN and WAN
To support testing scenarios where multiple Kubo instances run on the same machine, [`Routing.LoopbackAddressesOnLanDHT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingloopbackaddressesonlandht) is set to `true` when the `test` profile is applied.
#### Pin roots are now prioritized when announcing
The root CIDs of pinned content are now prioritized when announcing to the Amino DHT with [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) set to `all` (default) or `pinned`, making the important CIDs accessible faster.
### 📝 Changelog
### 👨‍👩‍👧‍👦 Contributors

View File

@ -1503,7 +1503,9 @@ Type: `optionalDuration` (unset for the default)
Tells reprovider what should be announced. Valid strategies are:
- `"all"` - announce all CIDs of stored blocks
- Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore
- `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks)
- Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
- `"roots"` - only announce the root block of explicitly pinned CIDs
- **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks.
It makes sense only for use cases where the entire DAG is fetched in full,
@ -1512,6 +1514,7 @@ Tells reprovider what should be announced. Valid strategies are:
providers for the missing block in the middle of a file, unless the peer
happens to already be connected to a provider and ask for child CID over
bitswap.
- `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything
Default: `"all"`

View File

@ -9,7 +9,7 @@ toolchain go1.22.0
replace github.com/ipfs/kubo => ./../../..
require (
github.com/ipfs/boxo v0.18.0
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.33.2
github.com/multiformats/go-multiaddr v0.12.3

View File

@ -266,8 +266,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=

2
go.mod
View File

@ -17,7 +17,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs-shipyard/nopfs v0.0.12
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
github.com/ipfs/boxo v0.18.0
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0

4
go.sum
View File

@ -329,8 +329,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

165
test/cli/provider_test.go Normal file
View File

@ -0,0 +1,165 @@
package cli
import (
"bytes"
"testing"
"time"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/ipfs/kubo/test/cli/testutils"
"github.com/stretchr/testify/require"
)
func TestProvider(t *testing.T) {
t.Parallel()
initNodes := func(t *testing.T, n int, fn func(n *harness.Node)) harness.Nodes {
nodes := harness.NewT(t).NewNodes(n).Init()
nodes.ForEachPar(fn)
return nodes.StartDaemons().Connect()
}
expectNoProviders := func(t *testing.T, cid string, nodes ...*harness.Node) {
for _, node := range nodes {
res := node.IPFS("routing", "findprovs", "-n=1", cid)
require.Empty(t, res.Stdout.String())
}
}
expectProviders := func(t *testing.T, cid, expectedProvider string, nodes ...*harness.Node) {
for _, node := range nodes {
res := node.IPFS("routing", "findprovs", "-n=1", cid)
require.Equal(t, expectedProvider, res.Stdout.Trimmed())
}
}
t.Run("Basic Providing", func(t *testing.T) {
t.Parallel()
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Experimental.StrategicProviding", false)
})
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String())
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
})
t.Run("Basic Strategic Providing", func(t *testing.T) {
t.Parallel()
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Experimental.StrategicProviding", true)
})
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String())
expectNoProviders(t, cid, nodes[1:]...)
})
t.Run("Reprovides with 'all' strategy", func(t *testing.T) {
t.Parallel()
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Reprovider.Strategy", "all")
})
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String(), "--local")
expectNoProviders(t, cid, nodes[1:]...)
nodes[0].IPFS("bitswap", "reprovide")
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
})
t.Run("Reprovides with 'flat' strategy", func(t *testing.T) {
t.Parallel()
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Reprovider.Strategy", "flat")
})
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String(), "--local")
expectNoProviders(t, cid, nodes[1:]...)
nodes[0].IPFS("bitswap", "reprovide")
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
})
t.Run("Reprovides with 'pinned' strategy", func(t *testing.T) {
t.Parallel()
foo := testutils.RandomBytes(1000)
bar := testutils.RandomBytes(1000)
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Reprovider.Strategy", "pinned")
})
defer nodes.StopDaemons()
cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo), "--offline", "--pin=false")
cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--offline", "--pin=false")
cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "--offline", "-w")
expectNoProviders(t, cidFoo, nodes[1:]...)
expectNoProviders(t, cidBar, nodes[1:]...)
expectNoProviders(t, cidBarDir, nodes[1:]...)
nodes[0].IPFS("bitswap", "reprovide")
expectNoProviders(t, cidFoo, nodes[1:]...)
expectProviders(t, cidBar, nodes[0].PeerID().String(), nodes[1:]...)
expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
})
t.Run("Reprovides with 'roots' strategy", func(t *testing.T) {
t.Parallel()
foo := testutils.RandomBytes(1000)
bar := testutils.RandomBytes(1000)
baz := testutils.RandomBytes(1000)
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Reprovider.Strategy", "roots")
})
defer nodes.StopDaemons()
cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo), "--offline", "--pin=false")
cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--offline", "--pin=false")
cidBaz := nodes[0].IPFSAdd(bytes.NewReader(baz), "--offline")
cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "--offline", "-w")
expectNoProviders(t, cidFoo, nodes[1:]...)
expectNoProviders(t, cidBar, nodes[1:]...)
expectNoProviders(t, cidBarDir, nodes[1:]...)
nodes[0].IPFS("bitswap", "reprovide")
expectNoProviders(t, cidFoo, nodes[1:]...)
expectNoProviders(t, cidBar, nodes[1:]...)
expectProviders(t, cidBaz, nodes[0].PeerID().String(), nodes[1:]...)
expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
})
t.Run("Providing works without ticking", func(t *testing.T) {
t.Parallel()
nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Reprovider.Interval", "0")
})
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String(), "--offline")
expectNoProviders(t, cid, nodes[1:]...)
nodes[0].IPFS("bitswap", "reprovide")
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
})
}

View File

@ -105,7 +105,7 @@ require (
github.com/hexops/gotextdiff v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/boxo v0.18.0 // indirect
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect

View File

@ -362,8 +362,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

View File

@ -1,34 +0,0 @@
#!/usr/bin/env bash
test_description="Test reprovider"
. lib/test-lib.sh
NUM_NODES=2
test_expect_success 'init iptb' '
iptb testbed create -type localipfs -force -count $NUM_NODES -init
'
test_expect_success 'peer ids' '
PEERID_0=$(iptb attr get 0 id) &&
PEERID_1=$(iptb attr get 1 id)
'
test_expect_success 'use strategic providing' '
iptb run -- ipfs config --json Experimental.StrategicProviding false
'
startup_cluster ${NUM_NODES}
test_expect_success 'add test object' '
HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q)
'
findprovs_expect '$HASH_0' '$PEERID_0'
test_expect_success 'stop node 1' '
iptb stop
'
test_done

View File

@ -1,140 +0,0 @@
#!/usr/bin/env bash
test_description="Test reprovider"
. lib/test-lib.sh
NUM_NODES=6
init_strategy() {
test_expect_success 'init iptb' '
iptb testbed create -type localipfs -force -count $NUM_NODES -init
'
test_expect_success 'peer ids' '
PEERID_0=$(iptb attr get 0 id) &&
PEERID_1=$(iptb attr get 1 id)
'
test_expect_success 'use pinning strategy for reprovider' '
ipfsi 0 config Reprovider.Strategy '$1'
'
startup_cluster ${NUM_NODES}
}
reprovide() {
test_expect_success 'reprovide' '
# TODO: this hangs, though only after reprovision was done
ipfsi 0 bitswap reprovide
'
}
# Test 'all' strategy
init_strategy 'all'
test_expect_success 'add test object' '
HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q --local)
'
findprovs_empty '$HASH_0'
reprovide
findprovs_expect '$HASH_0' '$PEERID_0'
test_expect_success 'Stop iptb' '
iptb stop
'
# Test 'pinned' strategy
init_strategy 'pinned'
test_expect_success 'prepare test files' '
date +"%FT%T.%N%z" > f1 &&
date +"%FT%T.%N%z" > f2
'
test_expect_success 'add test objects' '
HASH_FOO=$(ipfsi 0 add -q --offline --pin=false f1) &&
HASH_BAR=$(ipfsi 0 add -q --offline --pin=false f2) &&
HASH_BAR_DIR=$(ipfsi 0 add -q --offline -w f2)
'
findprovs_empty '$HASH_FOO'
findprovs_empty '$HASH_BAR'
findprovs_empty '$HASH_BAR_DIR'
reprovide
findprovs_empty '$HASH_FOO'
findprovs_expect '$HASH_BAR' '$PEERID_0'
findprovs_expect '$HASH_BAR_DIR' '$PEERID_0'
test_expect_success 'Stop iptb' '
iptb stop
'
# Test 'roots' strategy
init_strategy 'roots'
test_expect_success 'prepare test files' '
date +"%FT%T.%N%z" > f1 &&
date +"%FT%T.%N%z" > f2 &&
date +"%FT%T.%N%z" > f3
'
test_expect_success 'add test objects' '
HASH_FOO=$(ipfsi 0 add -q --offline --pin=false f1) &&
HASH_BAR=$(ipfsi 0 add -q --offline --pin=false f2) &&
HASH_BAZ=$(ipfsi 0 add -q --offline f3) &&
HASH_BAR_DIR=$(ipfsi 0 add -Q --offline -w f2)
'
findprovs_empty '$HASH_FOO'
findprovs_empty '$HASH_BAR'
findprovs_empty '$HASH_BAR_DIR'
reprovide
findprovs_empty '$HASH_FOO'
findprovs_empty '$HASH_BAR'
findprovs_expect '$HASH_BAZ' '$PEERID_0'
findprovs_expect '$HASH_BAR_DIR' '$PEERID_0'
test_expect_success 'Stop iptb' '
iptb stop
'
# Test reprovider working with ticking disabled
test_expect_success 'init iptb' '
iptb testbed create -type localipfs -force -count $NUM_NODES -init
'
test_expect_success 'peer ids' '
PEERID_0=$(iptb attr get 0 id) &&
PEERID_1=$(iptb attr get 1 id)
'
test_expect_success 'Disable reprovider ticking' '
ipfsi 0 config Reprovider.Interval 0
'
startup_cluster ${NUM_NODES}
test_expect_success 'add test object' '
HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q --offline)
'
findprovs_empty '$HASH_0'
reprovide
findprovs_expect '$HASH_0' '$PEERID_0'
test_expect_success 'resolve object $HASH_0' '
HASH_WITH_PREFIX=$(ipfsi 1 resolve $HASH_0)
'
findprovs_expect '$HASH_WITH_PREFIX' '$PEERID_0'
test_expect_success 'Stop iptb' '
iptb stop
'
test_done

View File

@ -1,34 +0,0 @@
#!/usr/bin/env bash
test_description="Test reprovider"
. lib/test-lib.sh
NUM_NODES=2
test_expect_success 'init iptb' '
iptb testbed create -type localipfs -force -count $NUM_NODES -init
'
test_expect_success 'peer ids' '
PEERID_0=$(iptb attr get 0 id) &&
PEERID_1=$(iptb attr get 1 id)
'
test_expect_success 'use strategic providing' '
iptb run -- ipfs config --json Experimental.StrategicProviding true
'
startup_cluster ${NUM_NODES}
test_expect_success 'add test object' '
HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q)
'
findprovs_empty '$HASH_0'
test_expect_success 'stop node 1' '
iptb stop
'
test_done