mirror of
https://github.com/ipfs/kubo.git
synced 2026-03-11 11:19:05 +08:00
Merge remote-tracking branch 'origin/master' into feat/get-closest-peers
This commit is contained in:
commit
0b07da9c94
@ -1,5 +1,6 @@
|
||||
# Kubo Changelogs
|
||||
|
||||
- [v0.40](docs/changelogs/v0.40.md)
|
||||
- [v0.39](docs/changelogs/v0.39.md)
|
||||
- [v0.38](docs/changelogs/v0.38.md)
|
||||
- [v0.37](docs/changelogs/v0.37.md)
|
||||
|
||||
@ -16,6 +16,8 @@ const (
|
||||
DefaultUnixFSRawLeaves = false
|
||||
DefaultUnixFSChunker = "size-262144"
|
||||
DefaultHashFunction = "sha2-256"
|
||||
DefaultFastProvideRoot = true
|
||||
DefaultFastProvideWait = false
|
||||
|
||||
DefaultUnixFSHAMTDirectorySizeThreshold = 262144 // 256KiB - https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26
|
||||
|
||||
@ -48,6 +50,8 @@ type Import struct {
|
||||
UnixFSHAMTDirectorySizeThreshold OptionalBytes
|
||||
BatchMaxNodes OptionalInteger
|
||||
BatchMaxSize OptionalInteger
|
||||
FastProvideRoot Flag
|
||||
FastProvideWait Flag
|
||||
}
|
||||
|
||||
// ValidateImportConfig validates the Import configuration according to UnixFS spec requirements.
|
||||
|
||||
@ -15,13 +15,18 @@ const (
|
||||
// DHT provider defaults
|
||||
DefaultProvideDHTInterval = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326
|
||||
DefaultProvideDHTMaxWorkers = 16 // Unified default for both sweep and legacy providers
|
||||
DefaultProvideDHTSweepEnabled = false
|
||||
DefaultProvideDHTSweepEnabled = true
|
||||
DefaultProvideDHTResumeEnabled = true
|
||||
DefaultProvideDHTDedicatedPeriodicWorkers = 2
|
||||
DefaultProvideDHTDedicatedBurstWorkers = 1
|
||||
DefaultProvideDHTMaxProvideConnsPerWorker = 20
|
||||
DefaultProvideDHTKeystoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes)
|
||||
DefaultProvideDHTOfflineDelay = 2 * time.Hour
|
||||
|
||||
// DefaultFastProvideTimeout is the maximum time allowed for fast-provide operations.
|
||||
// Prevents hanging on network issues when providing root CID.
|
||||
// 10 seconds is sufficient for DHT operations with sweep provider or accelerated client.
|
||||
DefaultFastProvideTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
type ProvideStrategy int
|
||||
@ -64,7 +69,7 @@ type ProvideDHT struct {
|
||||
MaxWorkers *OptionalInteger `json:",omitempty"`
|
||||
|
||||
// SweepEnabled activates the sweeping reprovider system which spreads
|
||||
// reprovide operations over time. This will become the default in a future release.
|
||||
// reprovide operations over time.
|
||||
// Default: DefaultProvideDHTSweepEnabled
|
||||
SweepEnabled Flag `json:",omitempty"`
|
||||
|
||||
@ -175,3 +180,25 @@ func ValidateProvideConfig(cfg *Provide) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShouldProvideForStrategy determines if content should be provided based on the provide strategy
|
||||
// and content characteristics (pinned status, root status, MFS status).
|
||||
func ShouldProvideForStrategy(strategy ProvideStrategy, isPinned bool, isPinnedRoot bool, isMFS bool) bool {
|
||||
if strategy == ProvideStrategyAll {
|
||||
// 'all' strategy: always provide
|
||||
return true
|
||||
}
|
||||
|
||||
// For combined strategies, check each component
|
||||
if strategy&ProvideStrategyPinned != 0 && isPinned {
|
||||
return true
|
||||
}
|
||||
if strategy&ProvideStrategyRoots != 0 && isPinnedRoot {
|
||||
return true
|
||||
}
|
||||
if strategy&ProvideStrategyMFS != 0 && isMFS {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@ -105,3 +105,87 @@ func TestValidateProvideConfig_MaxWorkers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldProvideForStrategy(t *testing.T) {
|
||||
t.Run("all strategy always provides", func(t *testing.T) {
|
||||
// ProvideStrategyAll should return true regardless of flags
|
||||
testCases := []struct{ pinned, pinnedRoot, mfs bool }{
|
||||
{false, false, false},
|
||||
{true, true, true},
|
||||
{true, false, false},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
assert.True(t, ShouldProvideForStrategy(
|
||||
ProvideStrategyAll, tc.pinned, tc.pinnedRoot, tc.mfs))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single strategies match only their flag", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
strategy ProvideStrategy
|
||||
pinned, pinnedRoot, mfs bool
|
||||
want bool
|
||||
}{
|
||||
{"pinned: matches when pinned=true", ProvideStrategyPinned, true, false, false, true},
|
||||
{"pinned: ignores other flags", ProvideStrategyPinned, false, true, true, false},
|
||||
|
||||
{"roots: matches when pinnedRoot=true", ProvideStrategyRoots, false, true, false, true},
|
||||
{"roots: ignores other flags", ProvideStrategyRoots, true, false, true, false},
|
||||
|
||||
{"mfs: matches when mfs=true", ProvideStrategyMFS, false, false, true, true},
|
||||
{"mfs: ignores other flags", ProvideStrategyMFS, true, true, false, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("combined strategies use OR logic (else-if bug fix)", func(t *testing.T) {
|
||||
// CRITICAL: Tests the fix where bitflag combinations (pinned+mfs) didn't work
|
||||
// because of else-if instead of separate if statements
|
||||
tests := []struct {
|
||||
name string
|
||||
strategy ProvideStrategy
|
||||
pinned, pinnedRoot, mfs bool
|
||||
want bool
|
||||
}{
|
||||
// pinned|mfs: provide if EITHER matches
|
||||
{"pinned|mfs when pinned", ProvideStrategyPinned | ProvideStrategyMFS, true, false, false, true},
|
||||
{"pinned|mfs when mfs", ProvideStrategyPinned | ProvideStrategyMFS, false, false, true, true},
|
||||
{"pinned|mfs when both", ProvideStrategyPinned | ProvideStrategyMFS, true, false, true, true},
|
||||
{"pinned|mfs when neither", ProvideStrategyPinned | ProvideStrategyMFS, false, false, false, false},
|
||||
|
||||
// roots|mfs
|
||||
{"roots|mfs when root", ProvideStrategyRoots | ProvideStrategyMFS, false, true, false, true},
|
||||
{"roots|mfs when mfs", ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true},
|
||||
{"roots|mfs when neither", ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false},
|
||||
|
||||
// pinned|roots
|
||||
{"pinned|roots when pinned", ProvideStrategyPinned | ProvideStrategyRoots, true, false, false, true},
|
||||
{"pinned|roots when root", ProvideStrategyPinned | ProvideStrategyRoots, false, true, false, true},
|
||||
{"pinned|roots when neither", ProvideStrategyPinned | ProvideStrategyRoots, false, false, false, false},
|
||||
|
||||
// triple combination
|
||||
{"all-three when any matches", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true},
|
||||
{"all-three when none match", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero strategy never provides", func(t *testing.T) {
|
||||
assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), false, false, false))
|
||||
assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), true, true, true))
|
||||
})
|
||||
}
|
||||
|
||||
@ -117,6 +117,16 @@ func (f Flag) String() string {
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveBoolFromConfig returns the resolved boolean value based on:
|
||||
// - If userSet is true, returns userValue (user explicitly set the flag)
|
||||
// - Otherwise, uses configFlag.WithDefault(defaultValue) (respects config or falls back to default)
|
||||
func ResolveBoolFromConfig(userValue bool, userSet bool, configFlag Flag, defaultValue bool) bool {
|
||||
if userSet {
|
||||
return userValue
|
||||
}
|
||||
return configFlag.WithDefault(defaultValue)
|
||||
}
|
||||
|
||||
var (
|
||||
_ json.Unmarshaler = (*Flag)(nil)
|
||||
_ json.Marshaler = (*Flag)(nil)
|
||||
|
||||
@ -61,20 +61,45 @@ const (
|
||||
inlineLimitOptionName = "inline-limit"
|
||||
toFilesOptionName = "to-files"
|
||||
|
||||
preserveModeOptionName = "preserve-mode"
|
||||
preserveMtimeOptionName = "preserve-mtime"
|
||||
modeOptionName = "mode"
|
||||
mtimeOptionName = "mtime"
|
||||
mtimeNsecsOptionName = "mtime-nsecs"
|
||||
preserveModeOptionName = "preserve-mode"
|
||||
preserveMtimeOptionName = "preserve-mtime"
|
||||
modeOptionName = "mode"
|
||||
mtimeOptionName = "mtime"
|
||||
mtimeNsecsOptionName = "mtime-nsecs"
|
||||
fastProvideRootOptionName = "fast-provide-root"
|
||||
fastProvideWaitOptionName = "fast-provide-wait"
|
||||
)
|
||||
|
||||
const adderOutChanSize = 8
|
||||
const (
|
||||
adderOutChanSize = 8
|
||||
)
|
||||
|
||||
var AddCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
Tagline: "Add a file or directory to IPFS.",
|
||||
ShortDescription: `
|
||||
Adds the content of <path> to IPFS. Use -r to add directories (recursively).
|
||||
|
||||
FAST PROVIDE OPTIMIZATION:
|
||||
|
||||
When you add content to IPFS, the sweep provider queues it for efficient
|
||||
DHT provides over time. While this is resource-efficient, other peers won't
|
||||
find your content immediately after 'ipfs add' completes.
|
||||
|
||||
To make sharing faster, 'ipfs add' does an immediate provide of the root CID
|
||||
to the DHT in addition to the regular queue. This complements the sweep provider:
|
||||
fast-provide handles the urgent case (root CIDs that users share and reference),
|
||||
while the sweep provider efficiently provides all blocks according to
|
||||
Provide.Strategy over time.
|
||||
|
||||
By default, this immediate provide runs in the background without blocking
|
||||
the command. If you need certainty that the root CID is discoverable before
|
||||
the command returns (e.g., sharing a link immediately), use --fast-provide-wait
|
||||
to wait for the provide to complete. Use --fast-provide-root=false to skip
|
||||
this optimization.
|
||||
|
||||
This works best with the sweep provider and accelerated DHT client.
|
||||
Automatically skipped when DHT is not available.
|
||||
`,
|
||||
LongDescription: `
|
||||
Adds the content of <path> to IPFS. Use -r to add directories.
|
||||
@ -213,6 +238,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"),
|
||||
cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). WARNING: experimental, forces dag-pb for root block, disables raw-leaves"),
|
||||
cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),
|
||||
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CID to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
|
||||
cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"),
|
||||
},
|
||||
PreRun: func(req *cmds.Request, env cmds.Environment) error {
|
||||
quiet, _ := req.Options[quietOptionName].(bool)
|
||||
@ -283,6 +310,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
mode, _ := req.Options[modeOptionName].(uint)
|
||||
mtime, _ := req.Options[mtimeOptionName].(int64)
|
||||
mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
|
||||
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
|
||||
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
|
||||
|
||||
if chunker == "" {
|
||||
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
|
||||
@ -319,6 +348,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout))
|
||||
}
|
||||
|
||||
fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot)
|
||||
fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait)
|
||||
|
||||
// Storing optional mode or mtime (UnixFS 1.5) requires root block
|
||||
// to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible.
|
||||
if preserveMode || preserveMtime || mode != 0 || mtime != 0 {
|
||||
@ -421,11 +453,12 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
}
|
||||
var added int
|
||||
var fileAddedToMFS bool
|
||||
var lastRootCid path.ImmutablePath // Track the root CID for fast-provide
|
||||
addit := toadd.Entries()
|
||||
for addit.Next() {
|
||||
_, dir := addit.Node().(files.Directory)
|
||||
errCh := make(chan error, 1)
|
||||
events := make(chan interface{}, adderOutChanSize)
|
||||
events := make(chan any, adderOutChanSize)
|
||||
opts[len(opts)-1] = options.Unixfs.Events(events)
|
||||
|
||||
go func() {
|
||||
@ -437,6 +470,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
return
|
||||
}
|
||||
|
||||
// Store the root CID for potential fast-provide operation
|
||||
lastRootCid = pathAdded
|
||||
|
||||
// creating MFS pointers when optional --to-files is set
|
||||
if toFilesSet {
|
||||
if addit.Name() == "" {
|
||||
@ -560,12 +596,29 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
return fmt.Errorf("expected a file argument")
|
||||
}
|
||||
|
||||
// Apply fast-provide-root if the flag is enabled
|
||||
if fastProvideRoot && (lastRootCid != path.ImmutablePath{}) {
|
||||
cfg, err := ipfsNode.Repo.Config()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmdenv.ExecuteFastProvide(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !fastProvideRoot {
|
||||
if fastProvideWait {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
PostRun: cmds.PostRunMap{
|
||||
cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
|
||||
sizeChan := make(chan int64, 1)
|
||||
outChan := make(chan interface{})
|
||||
outChan := make(chan any)
|
||||
req := res.Request()
|
||||
|
||||
// Could be slow.
|
||||
|
||||
@ -1,15 +1,19 @@
|
||||
package cmdenv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/kubo/commands"
|
||||
"github.com/ipfs/kubo/core"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
routing "github.com/libp2p/go-libp2p/core/routing"
|
||||
|
||||
"github.com/ipfs/kubo/commands"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/core"
|
||||
coreiface "github.com/ipfs/kubo/core/coreiface"
|
||||
options "github.com/ipfs/kubo/core/coreiface/options"
|
||||
)
|
||||
@ -86,3 +90,103 @@ func needEscape(s string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// provideCIDSync performs a synchronous/blocking provide operation to announce
|
||||
// the given CID to the DHT.
|
||||
//
|
||||
// - If the accelerated DHT client is used, a DHT lookup isn't needed, we
|
||||
// directly allocate provider records to closest peers.
|
||||
// - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an
|
||||
// optimistic provide call.
|
||||
// - Else we make a standard provide call (much slower).
|
||||
//
|
||||
// IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient()
|
||||
// before calling this function. Calling with a nil or invalid router will cause
|
||||
// a panic - this is the caller's responsibility to prevent.
|
||||
func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error {
|
||||
return router.Provide(ctx, c, true)
|
||||
}
|
||||
|
||||
// ExecuteFastProvide immediately provides a root CID to the DHT, bypassing the regular
|
||||
// provide queue for faster content discovery. This function is reusable across commands
|
||||
// that add or import content, such as ipfs add and ipfs dag import.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: context for synchronous provides
|
||||
// - ipfsNode: the IPFS node instance
|
||||
// - cfg: node configuration
|
||||
// - rootCid: the CID to provide
|
||||
// - wait: whether to block until provide completes (sync mode)
|
||||
// - isPinned: whether content is pinned
|
||||
// - isPinnedRoot: whether this is a pinned root CID
|
||||
// - isMFS: whether content is in MFS
|
||||
//
|
||||
// Return value:
|
||||
// - Returns nil if operation succeeded or was skipped (preconditions not met)
|
||||
// - Returns error only in sync mode (wait=true) when provide operation fails
|
||||
// - In async mode (wait=false), always returns nil (errors logged in goroutine)
|
||||
//
|
||||
// The function handles all precondition checks (Provide.Enabled, DHT availability,
|
||||
// strategy matching) and logs appropriately. In async mode, it launches a goroutine
|
||||
// with a detached context and timeout.
|
||||
func ExecuteFastProvide(
|
||||
ctx context.Context,
|
||||
ipfsNode *core.IpfsNode,
|
||||
cfg *config.Config,
|
||||
rootCid cid.Cid,
|
||||
wait bool,
|
||||
isPinned bool,
|
||||
isPinnedRoot bool,
|
||||
isMFS bool,
|
||||
) error {
|
||||
log.Debugw("fast-provide-root: enabled", "wait", wait)
|
||||
|
||||
// Check preconditions for providing
|
||||
switch {
|
||||
case !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled):
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "Provide.Enabled is false")
|
||||
return nil
|
||||
case cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0:
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "Provide.DHT.Interval is 0")
|
||||
return nil
|
||||
case !ipfsNode.HasActiveDHTClient():
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "DHT not available")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if strategy allows providing this content
|
||||
strategyStr := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
|
||||
strategy := config.ParseProvideStrategy(strategyStr)
|
||||
shouldProvide := config.ShouldProvideForStrategy(strategy, isPinned, isPinnedRoot, isMFS)
|
||||
|
||||
if !shouldProvide {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "strategy does not match content", "strategy", strategyStr, "pinned", isPinned, "pinnedRoot", isPinnedRoot, "mfs", isMFS)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Execute provide operation
|
||||
if wait {
|
||||
// Synchronous mode: block until provide completes, return error on failure
|
||||
log.Debugw("fast-provide-root: providing synchronously", "cid", rootCid)
|
||||
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
|
||||
log.Warnw("fast-provide-root: sync provide failed", "cid", rootCid, "error", err)
|
||||
return fmt.Errorf("fast-provide: %w", err)
|
||||
}
|
||||
log.Debugw("fast-provide-root: sync provide completed", "cid", rootCid)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Asynchronous mode (default): fire-and-forget, don't block, always return nil
|
||||
log.Debugw("fast-provide-root: providing asynchronously", "cid", rootCid)
|
||||
go func() {
|
||||
// Use detached context with timeout to prevent hanging on network issues
|
||||
ctx, cancel := context.WithTimeout(context.Background(), config.DefaultFastProvideTimeout)
|
||||
defer cancel()
|
||||
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
|
||||
log.Warnw("fast-provide-root: async provide failed", "cid", rootCid, "error", err)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: async provide completed", "cid", rootCid)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -74,10 +74,13 @@ func PathOrCidPath(str string) (path.Path, error) {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Save the original error before attempting fallback
|
||||
originalErr := err
|
||||
|
||||
if p, err := path.NewPath("/ipfs/" + str); err == nil {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Send back original err.
|
||||
return nil, err
|
||||
return nil, originalErr
|
||||
}
|
||||
|
||||
106
core/commands/cmdutils/utils_test.go
Normal file
106
core/commands/cmdutils/utils_test.go
Normal file
@ -0,0 +1,106 @@
|
||||
package cmdutils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPathOrCidPath(t *testing.T) {
|
||||
t.Run("valid path is returned as-is", func(t *testing.T) {
|
||||
validPath := "/ipfs/QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"
|
||||
p, err := PathOrCidPath(validPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, validPath, p.String())
|
||||
})
|
||||
|
||||
t.Run("valid CID is converted to /ipfs/ path", func(t *testing.T) {
|
||||
cid := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"
|
||||
p, err := PathOrCidPath(cid)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/ipfs/"+cid, p.String())
|
||||
})
|
||||
|
||||
t.Run("valid ipns path is returned as-is", func(t *testing.T) {
|
||||
validPath := "/ipns/example.com"
|
||||
p, err := PathOrCidPath(validPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, validPath, p.String())
|
||||
})
|
||||
|
||||
t.Run("returns original error when both attempts fail", func(t *testing.T) {
|
||||
invalidInput := "invalid!@#path"
|
||||
_, err := PathOrCidPath(invalidInput)
|
||||
require.Error(t, err)
|
||||
|
||||
// The error should reference the original input attempt.
|
||||
// This ensures users get meaningful error messages about their actual input.
|
||||
assert.Contains(t, err.Error(), invalidInput,
|
||||
"error should mention the original input")
|
||||
assert.Contains(t, err.Error(), "path does not have enough components",
|
||||
"error should describe the problem with the original input")
|
||||
})
|
||||
|
||||
t.Run("empty string returns error about original input", func(t *testing.T) {
|
||||
_, err := PathOrCidPath("")
|
||||
require.Error(t, err)
|
||||
|
||||
// Verify we're not getting an error about "/ipfs/" (the fallback)
|
||||
errMsg := err.Error()
|
||||
assert.NotContains(t, errMsg, "/ipfs/",
|
||||
"error should be about empty input, not the fallback path")
|
||||
})
|
||||
|
||||
t.Run("invalid characters return error about original input", func(t *testing.T) {
|
||||
invalidInput := "not a valid path or CID with spaces and /@#$%"
|
||||
_, err := PathOrCidPath(invalidInput)
|
||||
require.Error(t, err)
|
||||
|
||||
// The error message should help debug the original input
|
||||
assert.True(t, strings.Contains(err.Error(), invalidInput) ||
|
||||
strings.Contains(err.Error(), "invalid"),
|
||||
"error should reference original problematic input")
|
||||
})
|
||||
|
||||
t.Run("CID with path is converted correctly", func(t *testing.T) {
|
||||
cidWithPath := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG/file.txt"
|
||||
p, err := PathOrCidPath(cidWithPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/ipfs/"+cidWithPath, p.String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidatePinName(t *testing.T) {
|
||||
t.Run("valid pin name is accepted", func(t *testing.T) {
|
||||
err := ValidatePinName("my-pin-name")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("empty pin name is accepted", func(t *testing.T) {
|
||||
err := ValidatePinName("")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("pin name at max length is accepted", func(t *testing.T) {
|
||||
maxName := strings.Repeat("a", MaxPinNameBytes)
|
||||
err := ValidatePinName(maxName)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("pin name exceeding max length is rejected", func(t *testing.T) {
|
||||
tooLong := strings.Repeat("a", MaxPinNameBytes+1)
|
||||
err := ValidatePinName(tooLong)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "max")
|
||||
})
|
||||
|
||||
t.Run("pin name with unicode is counted by bytes", func(t *testing.T) {
|
||||
// Unicode character can be multiple bytes
|
||||
unicodeName := strings.Repeat("🔒", MaxPinNameBytes/4+1) // emoji is 4 bytes
|
||||
err := ValidatePinName(unicodeName)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "bytes")
|
||||
})
|
||||
}
|
||||
@ -16,10 +16,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
pinRootsOptionName = "pin-roots"
|
||||
progressOptionName = "progress"
|
||||
silentOptionName = "silent"
|
||||
statsOptionName = "stats"
|
||||
pinRootsOptionName = "pin-roots"
|
||||
progressOptionName = "progress"
|
||||
silentOptionName = "silent"
|
||||
statsOptionName = "stats"
|
||||
fastProvideRootOptionName = "fast-provide-root"
|
||||
fastProvideWaitOptionName = "fast-provide-wait"
|
||||
)
|
||||
|
||||
// DagCmd provides a subset of commands for interacting with ipld dag objects
|
||||
@ -189,6 +191,18 @@ Note:
|
||||
currently present in the blockstore does not represent a complete DAG,
|
||||
pinning of that individual root will fail.
|
||||
|
||||
FAST PROVIDE OPTIMIZATION:
|
||||
|
||||
Root CIDs from CAR headers are immediately provided to the DHT in addition
|
||||
to the regular provide queue, allowing other peers to discover your content
|
||||
right away. This complements the sweep provider, which efficiently provides
|
||||
all blocks according to Provide.Strategy over time.
|
||||
|
||||
By default, the provide happens in the background without blocking the
|
||||
command. Use --fast-provide-wait to wait for the provide to complete, or
|
||||
--fast-provide-root=false to skip it. Works even with --pin-roots=false.
|
||||
Automatically skipped when DHT is not available.
|
||||
|
||||
Maximum supported CAR version: 2
|
||||
Specification of CAR formats: https://ipld.io/specs/transport/car/
|
||||
`,
|
||||
@ -200,6 +214,8 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
|
||||
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
|
||||
cmds.BoolOption(silentOptionName, "No output."),
|
||||
cmds.BoolOption(statsOptionName, "Output stats."),
|
||||
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
|
||||
cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"),
|
||||
cmdutils.AllowBigBlockOption,
|
||||
},
|
||||
Type: CarImportOutput{},
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
ipldlegacy "github.com/ipfs/go-ipld-legacy"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/core/coreiface/options"
|
||||
gocarv2 "github.com/ipld/go-car/v2"
|
||||
@ -19,6 +20,8 @@ import (
|
||||
"github.com/ipfs/kubo/core/commands/cmdutils"
|
||||
)
|
||||
|
||||
var log = logging.Logger("core/commands")
|
||||
|
||||
func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
||||
node, err := cmdenv.GetNode(env)
|
||||
if err != nil {
|
||||
@ -47,6 +50,12 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
|
||||
doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
|
||||
|
||||
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
|
||||
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
|
||||
|
||||
fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot)
|
||||
fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait)
|
||||
|
||||
// grab a pinlock ( which doubles as a GC lock ) so that regardless of the
|
||||
// size of the streamed-in cars nothing will disappear on us before we had
|
||||
// a chance to roots that may show up at the very end
|
||||
@ -191,5 +200,21 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
}
|
||||
}
|
||||
|
||||
// Fast-provide roots for faster discovery
|
||||
if fastProvideRoot {
|
||||
err = roots.ForEach(func(c cid.Cid) error {
|
||||
return cmdenv.ExecuteFastProvide(req.Context, node, cfg, c, fastProvideWait, doPinRoots, doPinRoots, false)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if fastProvideWait {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -56,7 +56,7 @@ var queryDhtCmd = &cmds.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
if nd.DHTClient == nil {
|
||||
if !nd.HasActiveDHTClient() {
|
||||
return ErrNotDHT
|
||||
}
|
||||
|
||||
@ -70,7 +70,7 @@ var queryDhtCmd = &cmds.Command{
|
||||
ctx, events := routing.RegisterForQueryEvents(ctx)
|
||||
|
||||
client := nd.DHTClient
|
||||
if client == nd.DHT {
|
||||
if nd.DHT != nil && client == nd.DHT {
|
||||
client = nd.DHT.WAN
|
||||
if !nd.DHT.WANActive() {
|
||||
client = nd.DHT.LAN
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -11,6 +12,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
boxoprovider "github.com/ipfs/boxo/provider"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
"github.com/ipfs/kubo/core/commands/cmdenv"
|
||||
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
|
||||
@ -18,6 +20,7 @@ import (
|
||||
"github.com/libp2p/go-libp2p-kad-dht/provider/buffered"
|
||||
"github.com/libp2p/go-libp2p-kad-dht/provider/dual"
|
||||
"github.com/libp2p/go-libp2p-kad-dht/provider/stats"
|
||||
routing "github.com/libp2p/go-libp2p/core/routing"
|
||||
"github.com/probe-lab/go-libdht/kad/key"
|
||||
"golang.org/x/exp/constraints"
|
||||
)
|
||||
@ -575,3 +578,19 @@ func humanInt[T constraints.Integer](val T) string {
|
||||
func humanFull(val float64, decimals int) string {
|
||||
return humanize.CommafWithDigits(val, decimals)
|
||||
}
|
||||
|
||||
// provideCIDSync performs a synchronous/blocking provide operation to announce
|
||||
// the given CID to the DHT.
|
||||
//
|
||||
// - If the accelerated DHT client is used, a DHT lookup isn't needed, we
|
||||
// directly allocate provider records to closest peers.
|
||||
// - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an
|
||||
// optimistic provide call.
|
||||
// - Else we make a standard provide call (much slower).
|
||||
//
|
||||
// IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient()
|
||||
// before calling this function. Calling with a nil or invalid router will cause
|
||||
// a panic - this is the caller's responsibility to prevent.
|
||||
func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error {
|
||||
return router.Provide(ctx, c, true)
|
||||
}
|
||||
|
||||
@ -5,20 +5,22 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
oldcmds "github.com/ipfs/kubo/commands"
|
||||
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
|
||||
coreiface "github.com/ipfs/kubo/core/coreiface"
|
||||
corerepo "github.com/ipfs/kubo/core/corerepo"
|
||||
fsrepo "github.com/ipfs/kubo/repo/fsrepo"
|
||||
"github.com/ipfs/kubo/repo/fsrepo/migrations"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
bstore "github.com/ipfs/boxo/blockstore"
|
||||
"github.com/ipfs/boxo/path"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
)
|
||||
@ -226,45 +228,137 @@ Version string The repo version.
|
||||
},
|
||||
}
|
||||
|
||||
// VerifyProgress reports verification progress to the user.
|
||||
// It contains either a message about a corrupt block or a progress counter.
|
||||
type VerifyProgress struct {
|
||||
Msg string
|
||||
Progress int
|
||||
Msg string // Message about a corrupt/healed block (empty for valid blocks)
|
||||
Progress int // Number of blocks processed so far
|
||||
}
|
||||
|
||||
func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- string, bs bstore.Blockstore) {
|
||||
// verifyState represents the state of a block after verification.
|
||||
// States track both the verification result and any remediation actions taken.
|
||||
type verifyState int
|
||||
|
||||
const (
|
||||
verifyStateValid verifyState = iota // Block is valid and uncorrupted
|
||||
verifyStateCorrupt // Block is corrupt, no action taken
|
||||
verifyStateCorruptRemoved // Block was corrupt and successfully removed
|
||||
verifyStateCorruptRemoveFailed // Block was corrupt but removal failed
|
||||
verifyStateCorruptHealed // Block was corrupt, removed, and successfully re-fetched
|
||||
verifyStateCorruptHealFailed // Block was corrupt and removed, but re-fetching failed
|
||||
)
|
||||
|
||||
const (
|
||||
// verifyWorkerMultiplier determines worker pool size relative to CPU count.
|
||||
// Since block verification is I/O-bound (disk reads + potential network fetches),
|
||||
// we use more workers than CPU cores to maximize throughput.
|
||||
verifyWorkerMultiplier = 2
|
||||
)
|
||||
|
||||
// verifyResult contains the outcome of verifying a single block.
|
||||
// It includes the block's CID, its verification state, and an optional
|
||||
// human-readable message describing what happened.
|
||||
type verifyResult struct {
|
||||
cid cid.Cid // CID of the block that was verified
|
||||
state verifyState // Final state after verification and any remediation
|
||||
msg string // Human-readable message (empty for valid blocks)
|
||||
}
|
||||
|
||||
// verifyWorkerRun processes CIDs from the keys channel, verifying their integrity.
|
||||
// If shouldDrop is true, corrupt blocks are removed from the blockstore.
|
||||
// If shouldHeal is true (implies shouldDrop), removed blocks are re-fetched from the network.
|
||||
// The api parameter must be non-nil when shouldHeal is true.
|
||||
// healTimeout specifies the maximum time to wait for each block heal (0 = no timeout).
|
||||
func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- *verifyResult, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) {
|
||||
defer wg.Done()
|
||||
|
||||
sendResult := func(r *verifyResult) bool {
|
||||
select {
|
||||
case results <- r:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
for k := range keys {
|
||||
_, err := bs.Get(ctx, k)
|
||||
if err != nil {
|
||||
select {
|
||||
case results <- fmt.Sprintf("block %s was corrupt (%s)", k, err):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
// Block is corrupt
|
||||
result := &verifyResult{cid: k, state: verifyStateCorrupt}
|
||||
|
||||
if !shouldDrop {
|
||||
result.msg = fmt.Sprintf("block %s was corrupt (%s)", k, err)
|
||||
if !sendResult(result) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to delete
|
||||
if delErr := bs.DeleteBlock(ctx, k); delErr != nil {
|
||||
result.state = verifyStateCorruptRemoveFailed
|
||||
result.msg = fmt.Sprintf("block %s was corrupt (%s), failed to remove (%s)", k, err, delErr)
|
||||
if !sendResult(result) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if !shouldHeal {
|
||||
result.state = verifyStateCorruptRemoved
|
||||
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed", k, err)
|
||||
if !sendResult(result) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to heal by re-fetching from network (api is guaranteed non-nil here)
|
||||
healCtx := ctx
|
||||
var healCancel context.CancelFunc
|
||||
if healTimeout > 0 {
|
||||
healCtx, healCancel = context.WithTimeout(ctx, healTimeout)
|
||||
}
|
||||
|
||||
if _, healErr := api.Block().Get(healCtx, path.FromCid(k)); healErr != nil {
|
||||
result.state = verifyStateCorruptHealFailed
|
||||
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, failed to heal (%s)", k, err, healErr)
|
||||
} else {
|
||||
result.state = verifyStateCorruptHealed
|
||||
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, healed", k, err)
|
||||
}
|
||||
|
||||
if healCancel != nil {
|
||||
healCancel()
|
||||
}
|
||||
|
||||
if !sendResult(result) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case results <- "":
|
||||
case <-ctx.Done():
|
||||
// Block is valid
|
||||
if !sendResult(&verifyResult{cid: k, state: verifyStateValid}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore) <-chan string {
|
||||
results := make(chan string)
|
||||
// verifyResultChan creates a channel of verification results by spawning multiple worker goroutines
|
||||
// to process blocks in parallel. It returns immediately with a channel that will receive results.
|
||||
func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) <-chan *verifyResult {
|
||||
results := make(chan *verifyResult)
|
||||
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < runtime.NumCPU()*2; i++ {
|
||||
for i := 0; i < runtime.NumCPU()*verifyWorkerMultiplier; i++ {
|
||||
wg.Add(1)
|
||||
go verifyWorkerRun(ctx, &wg, keys, results, bs)
|
||||
go verifyWorkerRun(ctx, &wg, keys, results, bs, api, shouldDrop, shouldHeal, healTimeout)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
@ -276,6 +370,45 @@ func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blocks
|
||||
var repoVerifyCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
Tagline: "Verify all blocks in repo are not corrupted.",
|
||||
ShortDescription: `
|
||||
'ipfs repo verify' checks integrity of all blocks in the local datastore.
|
||||
Each block is read and validated against its CID to ensure data integrity.
|
||||
|
||||
Without any flags, this is a SAFE, read-only check that only reports corrupt
|
||||
blocks without modifying the repository. This can be used as a "dry run" to
|
||||
preview what --drop or --heal would do.
|
||||
|
||||
Use --drop to remove corrupt blocks, or --heal to remove and re-fetch from
|
||||
the network.
|
||||
|
||||
Examples:
|
||||
ipfs repo verify # safe read-only check, reports corrupt blocks
|
||||
ipfs repo verify --drop # remove corrupt blocks
|
||||
ipfs repo verify --heal # remove and re-fetch corrupt blocks
|
||||
|
||||
Exit Codes:
|
||||
0: All blocks are valid, OR all corrupt blocks were successfully remediated
|
||||
(with --drop or --heal)
|
||||
1: Corrupt blocks detected (without flags), OR remediation failed (block
|
||||
removal or healing failed with --drop or --heal)
|
||||
|
||||
Note: --heal requires the daemon to be running in online mode with network
|
||||
connectivity to nodes that have the missing blocks. Make sure the daemon is
|
||||
online and connected to other peers. Healing will attempt to re-fetch each
|
||||
corrupt block from the network after removing it. If a block cannot be found
|
||||
on the network, it will remain deleted.
|
||||
|
||||
WARNING: Both --drop and --heal are DESTRUCTIVE operations that permanently
|
||||
delete corrupt blocks from your repository. Once deleted, blocks cannot be
|
||||
recovered unless --heal successfully fetches them from the network. Blocks
|
||||
that cannot be healed will remain permanently deleted. Always backup your
|
||||
repository before using these options.
|
||||
`,
|
||||
},
|
||||
Options: []cmds.Option{
|
||||
cmds.BoolOption("drop", "Remove corrupt blocks from datastore (destructive operation)."),
|
||||
cmds.BoolOption("heal", "Remove corrupt blocks and re-fetch from network (destructive operation, implies --drop)."),
|
||||
cmds.StringOption("heal-timeout", "Maximum time to wait for each block heal (e.g., \"30s\"). Only applies with --heal.").WithDefault("30s"),
|
||||
},
|
||||
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
||||
nd, err := cmdenv.GetNode(env)
|
||||
@ -283,6 +416,38 @@ var repoVerifyCmd = &cmds.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
drop, _ := req.Options["drop"].(bool)
|
||||
heal, _ := req.Options["heal"].(bool)
|
||||
|
||||
if heal {
|
||||
drop = true // heal implies drop
|
||||
}
|
||||
|
||||
// Parse and validate heal-timeout
|
||||
timeoutStr, _ := req.Options["heal-timeout"].(string)
|
||||
healTimeout, err := time.ParseDuration(timeoutStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid heal-timeout: %w", err)
|
||||
}
|
||||
if healTimeout < 0 {
|
||||
return errors.New("heal-timeout must be >= 0")
|
||||
}
|
||||
|
||||
// Check online mode and API availability for healing operation
|
||||
var api coreiface.CoreAPI
|
||||
if heal {
|
||||
if !nd.IsOnline {
|
||||
return ErrNotOnline
|
||||
}
|
||||
api, err = cmdenv.GetApi(env, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if api == nil {
|
||||
return fmt.Errorf("healing requested but API is not available - make sure daemon is online and connected to other peers")
|
||||
}
|
||||
}
|
||||
|
||||
bs := &bstore.ValidatingBlockstore{Blockstore: bstore.NewBlockstore(nd.Repo.Datastore())}
|
||||
|
||||
keys, err := bs.AllKeysChan(req.Context)
|
||||
@ -291,17 +456,47 @@ var repoVerifyCmd = &cmds.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
results := verifyResultChan(req.Context, keys, bs)
|
||||
results := verifyResultChan(req.Context, keys, bs, api, drop, heal, healTimeout)
|
||||
|
||||
var fails int
|
||||
// Track statistics for each type of outcome
|
||||
var corrupted, removed, removeFailed, healed, healFailed int
|
||||
var i int
|
||||
for msg := range results {
|
||||
if msg != "" {
|
||||
if err := res.Emit(&VerifyProgress{Msg: msg}); err != nil {
|
||||
|
||||
for result := range results {
|
||||
// Update counters based on the block's final state
|
||||
switch result.state {
|
||||
case verifyStateCorrupt:
|
||||
// Block is corrupt but no action was taken (--drop not specified)
|
||||
corrupted++
|
||||
case verifyStateCorruptRemoved:
|
||||
// Block was corrupt and successfully removed (--drop specified)
|
||||
corrupted++
|
||||
removed++
|
||||
case verifyStateCorruptRemoveFailed:
|
||||
// Block was corrupt but couldn't be removed
|
||||
corrupted++
|
||||
removeFailed++
|
||||
case verifyStateCorruptHealed:
|
||||
// Block was corrupt, removed, and successfully re-fetched (--heal specified)
|
||||
corrupted++
|
||||
removed++
|
||||
healed++
|
||||
case verifyStateCorruptHealFailed:
|
||||
// Block was corrupt and removed, but re-fetching failed
|
||||
corrupted++
|
||||
removed++
|
||||
healFailed++
|
||||
default:
|
||||
// verifyStateValid blocks are not counted (they're the expected case)
|
||||
}
|
||||
|
||||
// Emit progress message for corrupt blocks
|
||||
if result.state != verifyStateValid && result.msg != "" {
|
||||
if err := res.Emit(&VerifyProgress{Msg: result.msg}); err != nil {
|
||||
return err
|
||||
}
|
||||
fails++
|
||||
}
|
||||
|
||||
i++
|
||||
if err := res.Emit(&VerifyProgress{Progress: i}); err != nil {
|
||||
return err
|
||||
@ -312,8 +507,42 @@ var repoVerifyCmd = &cmds.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
if fails != 0 {
|
||||
return errors.New("verify complete, some blocks were corrupt")
|
||||
if corrupted > 0 {
|
||||
// Build a summary of what happened with corrupt blocks
|
||||
summary := fmt.Sprintf("verify complete, %d blocks corrupt", corrupted)
|
||||
if removed > 0 {
|
||||
summary += fmt.Sprintf(", %d removed", removed)
|
||||
}
|
||||
if removeFailed > 0 {
|
||||
summary += fmt.Sprintf(", %d failed to remove", removeFailed)
|
||||
}
|
||||
if healed > 0 {
|
||||
summary += fmt.Sprintf(", %d healed", healed)
|
||||
}
|
||||
if healFailed > 0 {
|
||||
summary += fmt.Sprintf(", %d failed to heal", healFailed)
|
||||
}
|
||||
|
||||
// Determine success/failure based on operation mode
|
||||
shouldFail := false
|
||||
|
||||
if !drop {
|
||||
// Detection-only mode: always fail if corruption found
|
||||
shouldFail = true
|
||||
} else if heal {
|
||||
// Heal mode: fail if any removal or heal failed
|
||||
shouldFail = (removeFailed > 0 || healFailed > 0)
|
||||
} else {
|
||||
// Drop mode: fail if any removal failed
|
||||
shouldFail = (removeFailed > 0)
|
||||
}
|
||||
|
||||
if shouldFail {
|
||||
return errors.New(summary)
|
||||
}
|
||||
|
||||
// Success: emit summary as a message instead of error
|
||||
return res.Emit(&VerifyProgress{Msg: summary})
|
||||
}
|
||||
|
||||
return res.Emit(&VerifyProgress{Msg: "verify complete, all blocks validated."})
|
||||
@ -322,7 +551,7 @@ var repoVerifyCmd = &cmds.Command{
|
||||
Encoders: cmds.EncoderMap{
|
||||
cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *VerifyProgress) error {
|
||||
if strings.Contains(obj.Msg, "was corrupt") {
|
||||
fmt.Fprintln(os.Stdout, obj.Msg)
|
||||
fmt.Fprintln(w, obj.Msg)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
371
core/commands/repo_verify_test.go
Normal file
371
core/commands/repo_verify_test.go
Normal file
@ -0,0 +1,371 @@
|
||||
//go:build go1.25
|
||||
|
||||
package commands
|
||||
|
||||
// This file contains unit tests for the --heal-timeout flag functionality
|
||||
// using testing/synctest to avoid waiting for real timeouts.
|
||||
//
|
||||
// End-to-end tests for the full 'ipfs repo verify' command (including --drop
|
||||
// and --heal flags) are located in test/cli/repo_verify_test.go.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
coreiface "github.com/ipfs/kubo/core/coreiface"
|
||||
"github.com/ipfs/kubo/core/coreiface/options"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ipfs/boxo/path"
|
||||
)
|
||||
|
||||
func TestVerifyWorkerHealTimeout(t *testing.T) {
|
||||
t.Run("heal succeeds before timeout", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const healTimeout = 5 * time.Second
|
||||
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
|
||||
|
||||
// Setup channels
|
||||
keys := make(chan cid.Cid, 1)
|
||||
keys <- testCID
|
||||
close(keys)
|
||||
results := make(chan *verifyResult, 1)
|
||||
|
||||
// Mock blockstore that returns error (simulating corruption)
|
||||
mockBS := &mockBlockstore{
|
||||
getError: errors.New("corrupt block"),
|
||||
}
|
||||
|
||||
// Mock API where Block().Get() completes before timeout
|
||||
mockAPI := &mockCoreAPI{
|
||||
blockAPI: &mockBlockAPI{
|
||||
getDelay: 2 * time.Second, // Less than healTimeout
|
||||
data: []byte("healed data"),
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
// Run worker
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
|
||||
|
||||
// Advance time past the mock delay but before timeout
|
||||
time.Sleep(3 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
// Verify heal succeeded
|
||||
result := <-results
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, verifyStateCorruptHealed, result.state)
|
||||
assert.Contains(t, result.msg, "healed")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("heal fails due to timeout", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const healTimeout = 2 * time.Second
|
||||
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
|
||||
|
||||
// Setup channels
|
||||
keys := make(chan cid.Cid, 1)
|
||||
keys <- testCID
|
||||
close(keys)
|
||||
results := make(chan *verifyResult, 1)
|
||||
|
||||
// Mock blockstore that returns error (simulating corruption)
|
||||
mockBS := &mockBlockstore{
|
||||
getError: errors.New("corrupt block"),
|
||||
}
|
||||
|
||||
// Mock API where Block().Get() takes longer than healTimeout
|
||||
mockAPI := &mockCoreAPI{
|
||||
blockAPI: &mockBlockAPI{
|
||||
getDelay: 5 * time.Second, // More than healTimeout
|
||||
data: []byte("healed data"),
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
// Run worker
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
|
||||
|
||||
// Advance time past timeout
|
||||
time.Sleep(3 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
// Verify heal failed due to timeout
|
||||
result := <-results
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, verifyStateCorruptHealFailed, result.state)
|
||||
assert.Contains(t, result.msg, "failed to heal")
|
||||
assert.Contains(t, result.msg, "context deadline exceeded")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("heal with zero timeout still attempts heal", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const healTimeout = 0 // Zero timeout means no timeout
|
||||
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
|
||||
|
||||
// Setup channels
|
||||
keys := make(chan cid.Cid, 1)
|
||||
keys <- testCID
|
||||
close(keys)
|
||||
results := make(chan *verifyResult, 1)
|
||||
|
||||
// Mock blockstore that returns error (simulating corruption)
|
||||
mockBS := &mockBlockstore{
|
||||
getError: errors.New("corrupt block"),
|
||||
}
|
||||
|
||||
// Mock API that succeeds quickly
|
||||
mockAPI := &mockCoreAPI{
|
||||
blockAPI: &mockBlockAPI{
|
||||
getDelay: 100 * time.Millisecond,
|
||||
data: []byte("healed data"),
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
// Run worker
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
|
||||
|
||||
// Advance time to let heal complete
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
synctest.Wait()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
// Verify heal succeeded even with zero timeout
|
||||
result := <-results
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, verifyStateCorruptHealed, result.state)
|
||||
assert.Contains(t, result.msg, "healed")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("multiple blocks with different timeout outcomes", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const healTimeout = 3 * time.Second
|
||||
testCID1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
|
||||
testCID2 := cid.MustParse("bafybeihvvulpp4evxj7x7armbqcyg6uezzuig6jp3lktpbovlqfkjtgyby")
|
||||
|
||||
// Setup channels
|
||||
keys := make(chan cid.Cid, 2)
|
||||
keys <- testCID1
|
||||
keys <- testCID2
|
||||
close(keys)
|
||||
results := make(chan *verifyResult, 2)
|
||||
|
||||
// Mock blockstore that always returns error (all blocks corrupt)
|
||||
mockBS := &mockBlockstore{
|
||||
getError: errors.New("corrupt block"),
|
||||
}
|
||||
|
||||
// Create two mock block APIs with different delays
|
||||
// We'll need to alternate which one gets used
|
||||
// For simplicity, use one that succeeds fast
|
||||
mockAPI := &mockCoreAPI{
|
||||
blockAPI: &mockBlockAPI{
|
||||
getDelay: 1 * time.Second, // Less than healTimeout - will succeed
|
||||
data: []byte("healed data"),
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2) // Two workers
|
||||
|
||||
// Run two workers
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
|
||||
|
||||
// Advance time to let both complete
|
||||
time.Sleep(2 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
// Collect results
|
||||
var healedCount int
|
||||
for result := range results {
|
||||
if result.state == verifyStateCorruptHealed {
|
||||
healedCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Both should heal successfully (both under timeout)
|
||||
assert.Equal(t, 2, healedCount)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("valid block is not healed", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const healTimeout = 5 * time.Second
|
||||
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
|
||||
|
||||
// Setup channels
|
||||
keys := make(chan cid.Cid, 1)
|
||||
keys <- testCID
|
||||
close(keys)
|
||||
results := make(chan *verifyResult, 1)
|
||||
|
||||
// Mock blockstore that returns valid block (no error)
|
||||
mockBS := &mockBlockstore{
|
||||
block: blocks.NewBlock([]byte("valid data")),
|
||||
}
|
||||
|
||||
// Mock API (won't be called since block is valid)
|
||||
mockAPI := &mockCoreAPI{
|
||||
blockAPI: &mockBlockAPI{},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
// Run worker with heal enabled
|
||||
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, false, true, healTimeout)
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
// Verify block is marked valid, not healed
|
||||
result := <-results
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, verifyStateValid, result.state)
|
||||
assert.Empty(t, result.msg)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// mockBlockstore implements a minimal blockstore for testing
|
||||
type mockBlockstore struct {
|
||||
getError error
|
||||
block blocks.Block
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
|
||||
if m.getError != nil {
|
||||
return nil, m.getError
|
||||
}
|
||||
return m.block, nil
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
return m.block != nil, nil
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
if m.block != nil {
|
||||
return len(m.block.RawData()), nil
|
||||
}
|
||||
return 0, errors.New("block not found")
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) Put(ctx context.Context, b blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockBlockstore) HashOnRead(enabled bool) {
|
||||
}
|
||||
|
||||
// mockBlockAPI implements BlockAPI for testing
|
||||
type mockBlockAPI struct {
|
||||
getDelay time.Duration
|
||||
getError error
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (m *mockBlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) {
|
||||
if m.getDelay > 0 {
|
||||
select {
|
||||
case <-time.After(m.getDelay):
|
||||
// Delay completed
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if m.getError != nil {
|
||||
return nil, m.getError
|
||||
}
|
||||
return bytes.NewReader(m.data), nil
|
||||
}
|
||||
|
||||
func (m *mockBlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (coreiface.BlockStat, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockBlockAPI) Rm(ctx context.Context, p path.Path, opts ...options.BlockRmOption) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockBlockAPI) Stat(ctx context.Context, p path.Path) (coreiface.BlockStat, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// mockCoreAPI implements minimal CoreAPI for testing
|
||||
type mockCoreAPI struct {
|
||||
blockAPI *mockBlockAPI
|
||||
}
|
||||
|
||||
func (m *mockCoreAPI) Block() coreiface.BlockAPI {
|
||||
return m.blockAPI
|
||||
}
|
||||
|
||||
func (m *mockCoreAPI) Unixfs() coreiface.UnixfsAPI { return nil }
|
||||
func (m *mockCoreAPI) Dag() coreiface.APIDagService { return nil }
|
||||
func (m *mockCoreAPI) Name() coreiface.NameAPI { return nil }
|
||||
func (m *mockCoreAPI) Key() coreiface.KeyAPI { return nil }
|
||||
func (m *mockCoreAPI) Pin() coreiface.PinAPI { return nil }
|
||||
func (m *mockCoreAPI) Object() coreiface.ObjectAPI { return nil }
|
||||
func (m *mockCoreAPI) Swarm() coreiface.SwarmAPI { return nil }
|
||||
func (m *mockCoreAPI) PubSub() coreiface.PubSubAPI { return nil }
|
||||
func (m *mockCoreAPI) Routing() coreiface.RoutingAPI { return nil }
|
||||
|
||||
func (m *mockCoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) {
|
||||
return path.ImmutablePath{}, nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockCoreAPI) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockCoreAPI) WithOptions(...options.ApiOption) (coreiface.CoreAPI, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
@ -211,6 +211,10 @@ var provideRefRoutingCmd = &cmds.Command{
|
||||
ctx, events := routing.RegisterForQueryEvents(ctx)
|
||||
|
||||
var provideErr error
|
||||
// TODO: not sure if necessary to call StartProviding for `ipfs routing
|
||||
// provide <cid>`, since either cid is already being provided, or it will
|
||||
// be garbage collected and not reprovided anyway. So we may simply stick
|
||||
// with a single (optimistic) provide, and skip StartProviding call.
|
||||
go func() {
|
||||
defer cancel()
|
||||
if rec {
|
||||
@ -226,6 +230,16 @@ var provideRefRoutingCmd = &cmds.Command{
|
||||
}
|
||||
}()
|
||||
|
||||
if nd.HasActiveDHTClient() {
|
||||
// If node has a DHT client, provide immediately the supplied cids before
|
||||
// returning.
|
||||
for _, c := range cids {
|
||||
if err = provideCIDSync(req.Context, nd.DHTClient, c); err != nil {
|
||||
return fmt.Errorf("error providing cid: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for e := range events {
|
||||
if err := res.Emit(e); err != nil {
|
||||
return err
|
||||
@ -300,6 +314,7 @@ func provideCids(prov node.DHTProvider, cids []cid.Cid) error {
|
||||
for i, c := range cids {
|
||||
mhs[i] = c.Hash()
|
||||
}
|
||||
// providing happens asynchronously
|
||||
return prov.StartProviding(true, mhs...)
|
||||
}
|
||||
|
||||
|
||||
@ -75,7 +75,8 @@ This interface is not stable and may change from release to release.
|
||||
var dht *dht.IpfsDHT
|
||||
|
||||
var separateClient bool
|
||||
if nd.DHTClient != nd.DHT {
|
||||
// Check if using separate DHT client (e.g., accelerated DHT)
|
||||
if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT {
|
||||
separateClient = true
|
||||
}
|
||||
|
||||
|
||||
@ -255,7 +255,7 @@ func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutp
|
||||
}
|
||||
|
||||
// Amino DHT client keeps information about previously seen peers
|
||||
if nd.DHTClient != nd.DHT && nd.DHTClient != nil {
|
||||
if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT {
|
||||
client, ok := nd.DHTClient.(*fullrt.FullRT)
|
||||
if !ok {
|
||||
return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")
|
||||
|
||||
38
core/core.go
38
core/core.go
@ -30,9 +30,11 @@ import (
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
|
||||
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
psrouter "github.com/libp2p/go-libp2p-pubsub-router"
|
||||
record "github.com/libp2p/go-libp2p-record"
|
||||
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
|
||||
connmgr "github.com/libp2p/go-libp2p/core/connmgr"
|
||||
ic "github.com/libp2p/go-libp2p/core/crypto"
|
||||
p2phost "github.com/libp2p/go-libp2p/core/host"
|
||||
@ -143,6 +145,42 @@ func (n *IpfsNode) Close() error {
|
||||
return n.stop()
|
||||
}
|
||||
|
||||
// HasActiveDHTClient checks if the node's DHT client is active and usable for DHT operations.
|
||||
//
|
||||
// Returns false for:
|
||||
// - nil DHTClient
|
||||
// - typed nil pointers (e.g., (*ddht.DHT)(nil))
|
||||
// - no-op routers (routinghelpers.Null)
|
||||
//
|
||||
// Note: This method only checks for known DHT client types (ddht.DHT, fullrt.FullRT).
|
||||
// Custom routing.Routing implementations are not explicitly validated.
|
||||
//
|
||||
// This method prevents the "typed nil interface" bug where an interface contains
|
||||
// a nil pointer of a concrete type, which passes nil checks but panics when methods
|
||||
// are called.
|
||||
func (n *IpfsNode) HasActiveDHTClient() bool {
|
||||
if n.DHTClient == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for no-op router (Routing.Type=none)
|
||||
if _, ok := n.DHTClient.(routinghelpers.Null); ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for typed nil *ddht.DHT (common when Routing.Type=delegated or HTTP-only)
|
||||
if d, ok := n.DHTClient.(*ddht.DHT); ok && d == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for typed nil *fullrt.FullRT (accelerated DHT client)
|
||||
if f, ok := n.DHTClient.(*fullrt.FullRT); ok && f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Context returns the IpfsNode context
|
||||
func (n *IpfsNode) Context() context.Context {
|
||||
if n.ctx == nil {
|
||||
|
||||
@ -1,15 +1,28 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
context "context"
|
||||
|
||||
"github.com/ipfs/kubo/repo"
|
||||
|
||||
"github.com/ipfs/boxo/filestore"
|
||||
"github.com/ipfs/boxo/keystore"
|
||||
datastore "github.com/ipfs/go-datastore"
|
||||
syncds "github.com/ipfs/go-datastore/sync"
|
||||
config "github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/core/node/libp2p"
|
||||
golib "github.com/libp2p/go-libp2p"
|
||||
ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
|
||||
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
|
||||
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
pstore "github.com/libp2p/go-libp2p/core/peerstore"
|
||||
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
|
||||
)
|
||||
|
||||
func TestInitialization(t *testing.T) {
|
||||
@ -65,3 +78,151 @@ var testIdentity = config.Identity{
|
||||
PeerID: "QmNgdzLieYi8tgfo2WfTUzNVH5hQK9oAYGVf6dxN12NrHt",
|
||||
PrivKey: "CAASrRIwggkpAgEAAoICAQCwt67GTUQ8nlJhks6CgbLKOx7F5tl1r9zF4m3TUrG3Pe8h64vi+ILDRFd7QJxaJ/n8ux9RUDoxLjzftL4uTdtv5UXl2vaufCc/C0bhCRvDhuWPhVsD75/DZPbwLsepxocwVWTyq7/ZHsCfuWdoh/KNczfy+Gn33gVQbHCnip/uhTVxT7ARTiv8Qa3d7qmmxsR+1zdL/IRO0mic/iojcb3Oc/PRnYBTiAZFbZdUEit/99tnfSjMDg02wRayZaT5ikxa6gBTMZ16Yvienq7RwSELzMQq2jFA4i/TdiGhS9uKywltiN2LrNDBcQJSN02pK12DKoiIy+wuOCRgs2NTQEhU2sXCk091v7giTTOpFX2ij9ghmiRfoSiBFPJA5RGwiH6ansCHtWKY1K8BS5UORM0o3dYk87mTnKbCsdz4bYnGtOWafujYwzueGx8r+IWiys80IPQKDeehnLW6RgoyjszKgL/2XTyP54xMLSW+Qb3BPgDcPaPO0hmop1hW9upStxKsefW2A2d46Ds4HEpJEry7PkS5M4gKL/zCKHuxuXVk14+fZQ1rstMuvKjrekpAC2aVIKMI9VRA3awtnje8HImQMdj+r+bPmv0N8rTTr3eS4J8Yl7k12i95LLfK+fWnmUh22oTNzkRlaiERQrUDyE4XNCtJc0xs1oe1yXGqazCIAQIDAQABAoICAQCk1N/ftahlRmOfAXk//8wNl7FvdJD3le6+YSKBj0uWmN1ZbUSQk64chr12iGCOM2WY180xYjy1LOS44PTXaeW5bEiTSnb3b3SH+HPHaWCNM2EiSogHltYVQjKW+3tfH39vlOdQ9uQ+l9Gh6iTLOqsCRyszpYPqIBwi1NMLY2Ej8PpVU7ftnFWouHZ9YKS7nAEiMoowhTu/7cCIVwZlAy3AySTuKxPMVj9LORqC32PVvBHZaMPJ+X1Xyijqg6aq39WyoztkXg3+Xxx5j5eOrK6vO/Lp6ZUxaQilHDXoJkKEJjgIBDZpluss08UPfOgiWAGkW+L4fgUxY0qDLDAEMhyEBAn6KOKVL1JhGTX6GjhWziI94bddSpHKYOEIDzUy4H8BXnKhtnyQV6ELS65C2hj9D0IMBTj7edCF1poJy0QfdK0cuXgMvxHLeUO5uc2YWfbNosvKxqygB9rToy4b22YvNwsZUXsTY6Jt+p9V2OgXSKfB5VPeRbjTJL6xqvvUJpQytmII/C9JmSDUtCbYceHj6X9jgigLk20VV6nWHqCTj3utXD6NPAjoycVpLKDlnWEgfVELDIk0gobxUqqSm3jTPEKRPJgxkgPxbwxYumtw++1UY2y35w3WRDc2xYPaWKBCQeZy+mL6ByXp9bWlNvxS3Knb6oZp36/ovGnf2pGvdQKCAQEAyKpipz2lIUySDyE0avVWAmQb2tWGKXALPohzj7AwkcfEg2GuwoC6GyVE2sTJD1HRazIjOKn3yQORg2uOPeG7sx7EKHxSxCKDrbPawkvLCq8JYSy9TLvhqKUVVGYPqMBzu2POSLEA81QXas+aYjKOFWA2Zrjq26zV9ey3+6Lc6WULePgRQybU8+RHJc6fdjUCCfUxgOrUO2IQOuTJ+FsDpVnrMUGlokmWn23OjL4qTL9wGDnWGUs2pjSzNbj3qA0d8iqaiMUyHX/D/VS0wpeT1osNBSm8suvSibYBn+7wbIApbwXUxZaxMv2OHGz3empae4ckvNZs7r8wsI9UwFt8mwKCAQEA4XK6gZkv9t+3YCcSPw2ensLvL/xU7i2bkC9tfTGdjnQfzZXIf5KNdVuj/SerOl2S1s45NMs3ysJbADwRb4ahElD/V71nGzV8fpFTitC20ro9fuX4J0+twmBolHqeH9pmeGTjAeL1rvt6vxs4FkeG/yNft7GdXpXTtEGaObn8Mt0tPY+aB3UnKrnCQoQAlPyGHFrVRX0UEcp6wyyNGhJCNKeNOvqCHTFObhbhO+KWpWSN0MkVHnqaIBnIn1Te8FtvP/iTwXGnKc0YXJUG6+LM6LmOguW6tg8ZqiQeYyyR+e9eCFH4csLzkrTl1GxCxwEsoSLIMm7UDcjttW6tYEghkwKCAQEAmeCO5lCPYImnN5Lu71ZTLmI2OgmjaANTnBBnDbi+hgv61gUCToUIMejSdDCTPfwv61P3TmyIZs0luPGxkiKYHTNqmOE9Vspgz8Mr7fLRMNApESuNvloVIY32XVImj/GEzh4rAfM6F15U1sN8T/EUo6+0B/Glp+9R49QzAfRSE2g48/rGwgf1JVHYfVWFUtAzUA+GdqWdOixo5cCsYJbqpNHfWVZN/bUQnBFIYwUwysnC29D+LUdQEQQ4qOm+gFAOtrWU62zMkXJ4iLt8Ify6kbrvsRXgbhQIzzGS7WH9XDarj0eZciuslr15TLMC1Azadf+cXHLR9gMHA13mT9vYIQKCAQA/DjGv8cKCkAvf7s2hqROGYAs6Jp8yhrsN1tYOwAPLRhtnCs+rLrg17M2vDptLlcRuI/vIElamdTmylRpjUQpX7yObzLO73nfVhpwRJVMdGU394iBIDncQ+JoHfUwgqJskbUM40dvZdyjbrqc/Q/4z+hbZb+oN/GXb8sVKBATPzSDMKQ/xqgisYIw+wmDPStnPsHAaIWOtni47zIgilJzD0WEk78/YjmPbUrboYvWziK5JiRRJFA1rkQqV1c0M+OXixIm+/yS8AksgCeaHr0WUieGcJtjT9uE8vyFop5ykhRiNxy9wGaq6i7IEecsrkd6DqxDHWkwhFuO1bSE83q/VAoIBAEA+RX1i/SUi08p71ggUi9WFMqXmzELp1L3hiEjOc2AklHk2rPxsaTh9+G95BvjhP7fRa/Yga+yDtYuyjO99nedStdNNSg03aPXILl9gs3r2dPiQKUEXZJ3FrH6tkils/8BlpOIRfbkszrdZIKTO9GCdLWQ30dQITDACs8zV/1GFGrHFrqnnMe/NpIFHWNZJ0/WZMi8wgWO6Ik8jHEpQtVXRiXLqy7U6hk170pa4GHOzvftfPElOZZjy9qn7KjdAQqy6spIrAE94OEL+fBgbHQZGLpuTlj6w6YGbMtPU8uo7sXKoc6WOCb68JWft3tejGLDa1946HAWqVM9B/UcneNc=",
|
||||
}
|
||||
|
||||
// mockHostOption creates a HostOption that uses the provided mocknet.
|
||||
// Inlined to avoid import cycle with core/mock package.
|
||||
func mockHostOption(mn mocknet.Mocknet) libp2p.HostOption {
|
||||
return func(id peer.ID, ps pstore.Peerstore, opts ...golib.Option) (host.Host, error) {
|
||||
var cfg golib.Config
|
||||
if err := cfg.Apply(opts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The mocknet does not use the provided libp2p.Option. This options include
|
||||
// the listening addresses we want our peer listening on. Therefore, we have
|
||||
// to manually parse the configuration and add them here.
|
||||
ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL)
|
||||
return mn.AddPeerWithPeerstore(id, ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasActiveDHTClient(t *testing.T) {
|
||||
// Test 1: nil DHTClient
|
||||
t.Run("nil DHTClient", func(t *testing.T) {
|
||||
node := &IpfsNode{
|
||||
DHTClient: nil,
|
||||
}
|
||||
if node.HasActiveDHTClient() {
|
||||
t.Error("Expected false for nil DHTClient")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 2: Typed nil *ddht.DHT (common case when Routing.Type=delegated)
|
||||
t.Run("typed nil ddht.DHT", func(t *testing.T) {
|
||||
node := &IpfsNode{
|
||||
DHTClient: (*ddht.DHT)(nil),
|
||||
}
|
||||
if node.HasActiveDHTClient() {
|
||||
t.Error("Expected false for typed nil *ddht.DHT")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 3: Typed nil *fullrt.FullRT (accelerated DHT client)
|
||||
t.Run("typed nil fullrt.FullRT", func(t *testing.T) {
|
||||
node := &IpfsNode{
|
||||
DHTClient: (*fullrt.FullRT)(nil),
|
||||
}
|
||||
if node.HasActiveDHTClient() {
|
||||
t.Error("Expected false for typed nil *fullrt.FullRT")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 4: routinghelpers.Null no-op router (Routing.Type=none)
|
||||
t.Run("routinghelpers.Null", func(t *testing.T) {
|
||||
node := &IpfsNode{
|
||||
DHTClient: routinghelpers.Null{},
|
||||
}
|
||||
if node.HasActiveDHTClient() {
|
||||
t.Error("Expected false for routinghelpers.Null")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 5: Valid standard dual DHT (Routing.Type=auto/dht/dhtclient)
|
||||
t.Run("valid standard dual DHT", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mn := mocknet.New()
|
||||
defer mn.Close()
|
||||
|
||||
ds := syncds.MutexWrap(datastore.NewMapDatastore())
|
||||
c := config.Config{}
|
||||
c.Identity = testIdentity
|
||||
c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"}
|
||||
|
||||
r := &repo.Mock{
|
||||
C: c,
|
||||
D: ds,
|
||||
K: keystore.NewMemKeystore(),
|
||||
F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())),
|
||||
}
|
||||
|
||||
node, err := NewNode(ctx, &BuildCfg{
|
||||
Routing: libp2p.DHTServerOption,
|
||||
Repo: r,
|
||||
Host: mockHostOption(mn),
|
||||
Online: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create node with DHT: %v", err)
|
||||
}
|
||||
defer node.Close()
|
||||
|
||||
// First verify test setup created the expected DHT type
|
||||
if node.DHTClient == nil {
|
||||
t.Fatalf("Test setup failed: DHTClient is nil")
|
||||
}
|
||||
|
||||
if _, ok := node.DHTClient.(*ddht.DHT); !ok {
|
||||
t.Fatalf("Test setup failed: expected DHTClient to be *ddht.DHT, got %T", node.DHTClient)
|
||||
}
|
||||
|
||||
// Now verify HasActiveDHTClient() correctly identifies it as active
|
||||
if !node.HasActiveDHTClient() {
|
||||
t.Error("Expected true for valid dual DHT client")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 6: Valid accelerated DHT client (Routing.Type=autoclient)
|
||||
t.Run("valid accelerated DHT client", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mn := mocknet.New()
|
||||
defer mn.Close()
|
||||
|
||||
ds := syncds.MutexWrap(datastore.NewMapDatastore())
|
||||
c := config.Config{}
|
||||
c.Identity = testIdentity
|
||||
c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"}
|
||||
c.Routing.AcceleratedDHTClient = config.True
|
||||
|
||||
r := &repo.Mock{
|
||||
C: c,
|
||||
D: ds,
|
||||
K: keystore.NewMemKeystore(),
|
||||
F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())),
|
||||
}
|
||||
|
||||
node, err := NewNode(ctx, &BuildCfg{
|
||||
Routing: libp2p.DHTOption,
|
||||
Repo: r,
|
||||
Host: mockHostOption(mn),
|
||||
Online: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create node with accelerated DHT: %v", err)
|
||||
}
|
||||
defer node.Close()
|
||||
|
||||
// First verify test setup created the expected accelerated DHT type
|
||||
if node.DHTClient == nil {
|
||||
t.Fatalf("Test setup failed: DHTClient is nil")
|
||||
}
|
||||
|
||||
if _, ok := node.DHTClient.(*fullrt.FullRT); !ok {
|
||||
t.Fatalf("Test setup failed: expected DHTClient to be *fullrt.FullRT, got %T", node.DHTClient)
|
||||
}
|
||||
|
||||
// Now verify HasActiveDHTClient() correctly identifies it as active
|
||||
if !node.HasActiveDHTClient() {
|
||||
t.Error("Expected true for valid accelerated DHT client")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -240,14 +240,27 @@ func (tp *TestSuite) TestRoutingProvide(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
out, err = apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
maxAttempts := 5
|
||||
success := false
|
||||
for range maxAttempts {
|
||||
// We may need to try again as Provide() doesn't block until the CID is
|
||||
// actually provided.
|
||||
out, err = apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
provider := <-out
|
||||
|
||||
if provider.ID.String() == self0.ID().String() {
|
||||
success = true
|
||||
break
|
||||
}
|
||||
if len(provider.ID.String()) > 0 {
|
||||
t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String())
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
|
||||
provider := <-out
|
||||
|
||||
if provider.ID.String() != self0.ID().String() {
|
||||
t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String())
|
||||
if !success {
|
||||
t.Errorf("missing provider after %d attempts", maxAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
@ -55,12 +55,24 @@ func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (out P2PHo
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Optimistic provide is enabled either via dedicated expierimental flag, or when DHT Provide Sweep is enabled.
|
||||
// When DHT Provide Sweep is enabled, all provide operations go through the
|
||||
// `SweepingProvider`, hence the provides don't use the optimistic provide
|
||||
// logic. Provides use `SweepingProvider.StartProviding()` and not
|
||||
// `IpfsDHT.Provide()`, which is where the optimistic provide logic is
|
||||
// implemented. However, `IpfsDHT.Provide()` is used to quickly provide roots
|
||||
// when user manually adds content with the `--fast-provide` flag enabled. In
|
||||
// this case we want to use optimistic provide logic to quickly announce the
|
||||
// content to the network. This should be the only use case of
|
||||
// `IpfsDHT.Provide()` when DHT Provide Sweep is enabled.
|
||||
optimisticProvide := cfg.Experimental.OptimisticProvide || cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled)
|
||||
|
||||
routingOptArgs := RoutingOptionArgs{
|
||||
Ctx: ctx,
|
||||
Datastore: params.Repo.Datastore(),
|
||||
Validator: params.Validator,
|
||||
BootstrapPeers: bootstrappers,
|
||||
OptimisticProvide: cfg.Experimental.OptimisticProvide,
|
||||
OptimisticProvide: optimisticProvide,
|
||||
OptimisticProvideJobsPoolSize: cfg.Experimental.OptimisticProvideJobsPoolSize,
|
||||
LoopbackAddressesOnLanDHT: cfg.Routing.LoopbackAddressesOnLanDHT.WithDefault(config.DefaultLoopbackAddressesOnLanDHT),
|
||||
}
|
||||
|
||||
@ -116,6 +116,7 @@ type DHTProvider interface {
|
||||
// `OfflineDelay`). The schedule depends on the network size, hence recent
|
||||
// network connectivity is essential.
|
||||
RefreshSchedule() error
|
||||
Close() error
|
||||
}
|
||||
|
||||
var (
|
||||
@ -134,6 +135,7 @@ func (r *NoopProvider) StartProviding(bool, ...mh.Multihash) error { return nil
|
||||
func (r *NoopProvider) ProvideOnce(...mh.Multihash) error { return nil }
|
||||
func (r *NoopProvider) Clear() int { return 0 }
|
||||
func (r *NoopProvider) RefreshSchedule() error { return nil }
|
||||
func (r *NoopProvider) Close() error { return nil }
|
||||
|
||||
// LegacyProvider is a wrapper around the boxo/provider.System that implements
|
||||
// the DHTProvider interface. This provider manages reprovides using a burst
|
||||
@ -523,8 +525,41 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
// Keystore data isn't purged, on close, but it will be overwritten
|
||||
// when the node starts again.
|
||||
// Keystore will be closed by ensureProviderClosesBeforeKeystore hook
|
||||
// to guarantee provider closes before keystore.
|
||||
return nil
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ensureProviderClosesBeforeKeystore manages the shutdown order between
|
||||
// provider and keystore to prevent race conditions.
|
||||
//
|
||||
// The provider's worker goroutines may call keystore methods during their
|
||||
// operation. If keystore closes while these operations are in-flight, we get
|
||||
// "keystore is closed" errors. By closing the provider first, we ensure all
|
||||
// worker goroutines exit and complete any pending keystore operations before
|
||||
// the keystore itself closes.
|
||||
type providerKeystoreShutdownInput struct {
|
||||
fx.In
|
||||
Provider DHTProvider
|
||||
Keystore *keystore.ResettableKeystore
|
||||
}
|
||||
ensureProviderClosesBeforeKeystore := fx.Invoke(func(lc fx.Lifecycle, in providerKeystoreShutdownInput) {
|
||||
// Skip for NoopProvider
|
||||
if _, ok := in.Provider.(*NoopProvider); ok {
|
||||
return
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
// Close provider first - waits for all worker goroutines to exit.
|
||||
// This ensures no code can access keystore after this returns.
|
||||
if err := in.Provider.Close(); err != nil {
|
||||
logger.Errorw("error closing provider during shutdown", "error", err)
|
||||
}
|
||||
|
||||
// Close keystore - safe now, provider is fully shut down
|
||||
return in.Keystore.Close()
|
||||
},
|
||||
})
|
||||
@ -650,6 +685,7 @@ See docs: https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtmaxw
|
||||
return fx.Options(
|
||||
sweepingReprovider,
|
||||
initKeystore,
|
||||
ensureProviderClosesBeforeKeystore,
|
||||
reprovideAlert,
|
||||
)
|
||||
}
|
||||
|
||||
@ -59,6 +59,9 @@ A new experimental DHT provider is available as an alternative to both the defau
|
||||
|
||||
**Monitoring and debugging:** Legacy mode (`SweepEnabled=false`) tracks `provider_reprovider_provide_count` and `provider_reprovider_reprovide_count`, while sweep mode (`SweepEnabled=true`) tracks `total_provide_count_total`. Enable debug logging with `GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug` to see detailed logs from either system.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly.
|
||||
|
||||
> [!NOTE]
|
||||
> This feature is experimental and opt-in. In the future, it will become the default and replace the legacy system. Some commands like `ipfs stats provide` and `ipfs routing provide` are not yet available with sweep mode. Run `ipfs provide --help` for alternatives.
|
||||
|
||||
@ -68,6 +71,9 @@ For configuration details, see [`Provide.DHT`](https://github.com/ipfs/kubo/blob
|
||||
|
||||
Kubo now exposes DHT metrics from [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/), including `total_provide_count_total` for sweep provider operations and RPC metrics prefixed with `rpc_inbound_` and `rpc_outbound_` for DHT message traffic. See [Kubo metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md) for details.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly.
|
||||
|
||||
#### 🚨 Improved gateway error pages with diagnostic tools
|
||||
|
||||
Gateway error pages now provide more actionable information during content retrieval failures. When a 504 Gateway Timeout occurs, users see detailed retrieval state information including which phase failed and a sample of providers that were attempted:
|
||||
|
||||
@ -10,11 +10,15 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
|
||||
|
||||
- [Overview](#overview)
|
||||
- [🔦 Highlights](#-highlights)
|
||||
- [🎯 Amino DHT Sweep provider is now the default](#-amino-dht-sweep-provider-is-now-the-default)
|
||||
- [⚡ Fast root CID providing for immediate content discovery](#-fast-root-cid-providing-for-immediate-content-discovery)
|
||||
- [📊 Detailed statistics for Sweep provider with `ipfs provide stat`](#-detailed-statistics-for-sweep-provider-with-ipfs-provide-stat)
|
||||
- [⏯️ Provider resume cycle for improved reproviding reliability](#provider-resume-cycle-for-improved-reproviding-reliability)
|
||||
- [🔔 Sweep provider slow reprovide warnings](#-sweep-provider-slow-reprovide-warnings)
|
||||
- [📊 Metric rename: `provider_provides_total`](#-metric-rename-provider_provides_total)
|
||||
- [🔧 Fixed UPnP port forwarding after router restarts](#-fixed-upnp-port-forwarding-after-router-restarts)
|
||||
- [🖥️ RISC-V support with prebuilt binaries](#️-risc-v-support-with-prebuilt-binaries)
|
||||
- [🚦 Gateway range request limits for CDN compatibility](#-gateway-range-request-limits-for-cdn-compatibility)
|
||||
- [🪦 Deprecated `go-ipfs` name no longer published](#-deprecated-go-ipfs-name-no-longer-published)
|
||||
- [📦️ Important dependency updates](#-important-dependency-updates)
|
||||
- [📝 Changelog](#-changelog)
|
||||
@ -22,77 +26,77 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
|
||||
|
||||
### Overview
|
||||
|
||||
Kubo 0.39.0 graduates the experimental sweep provider to default, bringing efficient content announcement to all nodes. This release adds fast root CID providing for immediate content discovery via `ipfs add`, detailed provider statistics, automatic state persistence for reliable reproviding after restarts, and proactive monitoring alerts for identifying issues early. It also includes important fixes for UPnP port forwarding, RISC-V prebuilt binaries, and finalizes the deprecation of the legacy go-ipfs name.
|
||||
|
||||
### 🔦 Highlights
|
||||
|
||||
#### 🚦 Gateway range request limits for CDN compatibility
|
||||
#### 🎯 Amino DHT Sweep provider is now the default
|
||||
|
||||
The new [`Gateway.MaxRangeRequestFileSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) configuration protects against CDN bugs where range requests over a certain size are silently ignored and the entire file is returned instead ([boxo#856](https://github.com/ipfs/boxo/issues/856#issuecomment-2786431369)). This causes unexpected bandwidth costs for both gateway operators and clients who only wanted a small byte range.
|
||||
The Amino DHT Sweep provider system, introduced as experimental in v0.38, is now enabled by default (`Provide.DHT.SweepEnabled=true`).
|
||||
|
||||
**What this means:** All nodes now benefit from efficient keyspace-sweeping content announcements that reduce memory overhead and create predictable network patterns, especially for nodes providing large content collections.
|
||||
|
||||
**Migration:** The transition is automatic on upgrade. Your existing configuration is preserved:
|
||||
|
||||
- If you explicitly set `Provide.DHT.SweepEnabled=false` in v0.38, you'll continue using the legacy provider
|
||||
- If you were using the default settings, you'll automatically get the sweep provider
|
||||
- To opt out and return to legacy behavior: `ipfs config --json Provide.DHT.SweepEnabled false`
|
||||
|
||||
**New features available with sweep mode:**
|
||||
|
||||
- Detailed statistics via `ipfs provide stat` ([see below](#-detailed-statistics-for-sweep-provider-with-ipfs-provide-stat))
|
||||
- Automatic resume after restarts with persistent state ([see below](#provider-resume-cycle-for-improved-reproviding-reliability))
|
||||
- Proactive alerts when reproviding falls behind ([see below](#-sweep-provider-slow-reprovide-warnings))
|
||||
- Better metrics for monitoring (`provider_provides_total`) ([see below](#-metric-rename-provider_provides_total))
|
||||
- Fast optimistic provide of new root CIDs ([see below](#-fast-root-cid-providing-for-immediate-content-discovery))
|
||||
|
||||
For background on the sweep provider design and motivations, see [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled) and [ipshipyard.com#8](https://github.com/ipshipyard/ipshipyard.com/pull/8).
|
||||
|
||||
#### ⚡ Fast root CID providing for immediate content discovery
|
||||
|
||||
When you add content to IPFS, the sweep provider queues it for efficient DHT provides over time. While this is resource-efficient, other peers won't find your content immediately after `ipfs add` or `ipfs dag import` completes.
|
||||
|
||||
To make sharing faster, `ipfs add` and `ipfs dag import` now do an immediate provide of root CIDs to the DHT in addition to the regular queue (controlled by the new `--fast-provide-root` flag, enabled by default). This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to `Provide.Strategy` over time.
|
||||
|
||||
This closes the gap between command completion and content shareability: root CIDs typically become discoverable on the network in under a second (compared to 30+ seconds previously). The feature uses optimistic DHT operations, which are significantly faster with the sweep provider (now enabled by default).
|
||||
|
||||
By default, this immediate provide runs in the background without blocking the command. For use cases requiring guaranteed discoverability before the command returns (e.g., sharing a link immediately), use `--fast-provide-wait` to block until the provide completes.
|
||||
|
||||
**Simple examples:**
|
||||
|
||||
```bash
|
||||
ipfs add file.txt # Root provided immediately, blocks queued for sweep provider
|
||||
ipfs add file.txt --fast-provide-wait # Wait for root provide to complete
|
||||
ipfs dag import file.car # Same for CAR imports
|
||||
```
|
||||
|
||||
**Configuration:** Set defaults via `Import.FastProvideRoot` (default: `true`) and `Import.FastProvideWait` (default: `false`). See `ipfs add --help` and `ipfs dag import --help` for more details and examples.
|
||||
|
||||
This optimization works best with the sweep provider and accelerated DHT client, where provide operations are significantly faster. Automatically skipped when DHT is unavailable (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
Set this to your CDN's range request limit (e.g., `"5GiB"` for Cloudflare's default plan) to return 501 Not Implemented for oversized range requests, with an error message suggesting verifiable block requests as an alternative.
|
||||
#### 📊 Detailed statistics for Sweep provider with `ipfs provide stat`
|
||||
|
||||
The experimental Sweep provider system ([introduced in
|
||||
v0.38](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.38.md#-experimental-sweeping-dht-provider))
|
||||
now has detailed statistics available through `ipfs provide stat`.
|
||||
The Sweep provider system now exposes detailed statistics through `ipfs provide stat`, helping you monitor provider health and troubleshoot issues.
|
||||
|
||||
These statistics help you monitor provider health and troubleshoot issues,
|
||||
especially useful for nodes providing large content collections. You can quickly
|
||||
identify bottlenecks like queue backlog, worker saturation, or connectivity
|
||||
problems that might prevent content from being announced to the DHT.
|
||||
Run `ipfs provide stat` for a quick summary, or use `--all` to see complete metrics including connectivity status, queue sizes, reprovide schedules, network statistics, operation rates, and worker utilization. For real-time monitoring, use `watch ipfs provide stat --all --compact` to observe changes in a 2-column layout. Individual sections can be displayed with flags like `--network`, `--operations`, or `--workers`.
|
||||
|
||||
**Default behavior:** Displays a brief summary showing queue sizes, scheduled
|
||||
CIDs/regions, average record holders, ongoing/total provides, and worker status
|
||||
when resources are constrained.
|
||||
For Dual DHT configurations, use `--lan` to view LAN DHT statistics instead of the default WAN DHT stats.
|
||||
|
||||
**Detailed statistics with `--all`:** View complete metrics organized into sections:
|
||||
|
||||
- **Connectivity**: DHT connection status
|
||||
- **Queues**: Pending provide and reprovide operations
|
||||
- **Schedule**: CIDs/regions scheduled for reprovide
|
||||
- **Timings**: Uptime, reprovide cycle information
|
||||
- **Network**: Peer statistics, keyspace region sizes
|
||||
- **Operations**: Ongoing and past provides, rates, errors
|
||||
- **Workers**: Worker pool utilization and availability
|
||||
|
||||
**Real-time monitoring:** For continuous monitoring, run
|
||||
`watch ipfs provide stat --all --compact` to see detailed statistics refreshed
|
||||
in a 2-column layout. This lets you observe provide rates, queue sizes, and
|
||||
worker availability in real-time. Individual sections can be displayed using
|
||||
flags like `--network`, `--operations`, or `--workers`, and multiple flags can
|
||||
be combined for custom views.
|
||||
|
||||
**Dual DHT support:** For Dual DHT configurations, use `--lan` to view LAN DHT
|
||||
provider statistics instead of the default WAN DHT stats.
|
||||
For more information, run `ipfs provide stat --help` or see the [Provide Stats documentation](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md).
|
||||
|
||||
> [!NOTE]
|
||||
> These statistics are only available when using the Sweep provider system
|
||||
> (enabled via
|
||||
> [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled)).
|
||||
> Legacy provider shows basic statistics without flag support.
|
||||
> Legacy provider (when `Provide.DHT.SweepEnabled=false`) shows basic statistics without flag support.
|
||||
|
||||
#### ⏯️ Provider resume cycle for improved reproviding reliability
|
||||
|
||||
When using the sweeping provider (`Provide.DHT.SweepEnabled`), Kubo now
|
||||
persists the reprovide cycle state and automatically resumes where it left off
|
||||
after a restart. This brings several improvements:
|
||||
The Sweep provider now persists the reprovide cycle state and automatically resumes where it left off after a restart. This brings several improvements:
|
||||
|
||||
- **Persistent progress**: The provider now saves its position in the reprovide
|
||||
cycle to the datastore. On restart, it continues from where it stopped instead
|
||||
of starting from scratch.
|
||||
- **Catch-up reproviding**: If the node was offline for an extended period, all
|
||||
CIDs that haven't been reprovided within the configured reprovide interval are
|
||||
immediately queued for reproviding when the node starts up. This ensures
|
||||
content availability is maintained even after downtime.
|
||||
- **Persistent provide queue**: The provide queue is now persisted to the
|
||||
datastore on shutdown. When the node restarts, queued CIDs are restored and
|
||||
provided as expected, preventing loss of pending provide operations.
|
||||
- **Resume control**: The resume behavior is now controlled via the
|
||||
`Provide.DHT.ResumeEnabled` config option (default: `true`). If you don't want
|
||||
to keep the persisted provider state from a previous run, you can set
|
||||
`Provide.DHT.ResumeEnabled=false` in your config.
|
||||
- **Persistent progress**: The provider saves its position in the reprovide cycle to the datastore. On restart, it continues from where it stopped instead of starting from scratch.
|
||||
- **Catch-up reproviding**: If the node was offline for an extended period, all CIDs that haven't been reprovided within the configured reprovide interval are immediately queued for reproviding when the node starts up. This ensures content availability is maintained even after downtime.
|
||||
- **Persistent provide queue**: The provide queue is persisted to the datastore on shutdown. When the node restarts, queued CIDs are restored and provided as expected, preventing loss of pending provide operations.
|
||||
- **Resume control**: The resume behavior is controlled via [`Provide.DHT.ResumeEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtresumeenabled) (default: `true`). Set to `false` if you don't want to keep the persisted provider state from a previous run.
|
||||
|
||||
This feature significantly improves the reliability of content providing,
|
||||
especially for nodes that experience intermittent connectivity or restarts.
|
||||
This feature improves reliability for nodes that experience intermittent connectivity or restarts.
|
||||
|
||||
#### 🔔 Sweep provider slow reprovide warnings
|
||||
|
||||
@ -110,6 +114,12 @@ The alert polls every 15 minutes (to avoid alert fatigue while catching
|
||||
persistent issues) and only triggers after sustained growth across multiple
|
||||
intervals. The legacy provider is unaffected by this change.
|
||||
|
||||
#### 📊 Metric rename: `provider_provides_total`
|
||||
|
||||
The Amino DHT Sweep provider metric has been renamed from `total_provide_count_total` to `provider_provides_total` to follow OpenTelemetry naming conventions and maintain consistency with other kad-dht metrics (which use dot notation like `rpc.inbound.messages`, `rpc.outbound.requests`, etc.).
|
||||
|
||||
**Migration:** If you have Prometheus queries, dashboards, or alerts monitoring the old `total_provide_count_total` metric, update them to use `provider_provides_total` instead. This affects all nodes using sweep mode, which is now the default in v0.39 (previously opt-in experimental in v0.38).
|
||||
|
||||
#### 🔧 Fixed UPnP port forwarding after router restarts
|
||||
|
||||
Kubo now automatically recovers UPnP port mappings when routers restart or
|
||||
@ -136,26 +146,27 @@ using UPnP for NAT traversal.
|
||||
|
||||
#### 🖥️ RISC-V support with prebuilt binaries
|
||||
|
||||
Kubo now provides official `linux-riscv64` prebuilt binaries with every release,
|
||||
bringing IPFS to [RISC-V](https://en.wikipedia.org/wiki/RISC-V) open hardware.
|
||||
Kubo provides official `linux-riscv64` prebuilt binaries, bringing IPFS to [RISC-V](https://en.wikipedia.org/wiki/RISC-V) open hardware.
|
||||
|
||||
As RISC-V single-board computers and embedded systems become more accessible,
|
||||
it's good to see the distributed web supported on open hardware architectures -
|
||||
a natural pairing of open technologies.
|
||||
As RISC-V single-board computers and embedded systems become more accessible, the distributed web is now supported on open hardware architectures - a natural pairing of open technologies.
|
||||
|
||||
Download from <https://dist.ipfs.tech/kubo/> or
|
||||
<https://github.com/ipfs/kubo/releases> and look for the `linux-riscv64` archive.
|
||||
Download from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases> and look for the `linux-riscv64` archive.
|
||||
|
||||
#### 🚦 Gateway range request limits for CDN compatibility
|
||||
|
||||
The new [`Gateway.MaxRangeRequestFileSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) configuration protects against CDN range request limitations that cause bandwidth overcharges on deserialized responses. Some CDNs convert range requests over large files into full file downloads, causing clients requesting small byte ranges to unknowingly download entire multi-gigabyte files.
|
||||
|
||||
This only impacts deserialized responses. Clients using verifiable block requests (`application/vnd.ipld.raw`) are not affected. See the [configuration documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) for details.
|
||||
|
||||
#### 🪦 Deprecated `go-ipfs` name no longer published
|
||||
|
||||
The `go-ipfs` name was deprecated in 2022 and renamed to `kubo`. Starting with this release, we have stopped publishing Docker images and distribution binaries under the old `go-ipfs` name.
|
||||
The `go-ipfs` name was deprecated in 2022 and renamed to `kubo`. Starting with this release, the legacy Docker image name has been replaced with a stub that displays an error message directing users to switch to `ipfs/kubo`.
|
||||
|
||||
Existing users should switch to:
|
||||
**Docker images:** The `ipfs/go-ipfs` image tags now contain only a stub script that exits with an error, instructing users to update their Docker configurations to use [`ipfs/kubo`](https://hub.docker.com/r/ipfs/kubo) instead. This ensures users are aware of the deprecation while allowing existing automation to fail explicitly rather than silently using outdated images.
|
||||
|
||||
- Docker: `ipfs/kubo` image (instead of `ipfs/go-ipfs`)
|
||||
- Binaries: download from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases>
|
||||
**Distribution binaries:** Download Kubo from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases>. The legacy `go-ipfs` distribution path should no longer be used.
|
||||
|
||||
For Docker users, the legacy `ipfs/go-ipfs` image name now shows a deprecation notice directing you to `ipfs/kubo`.
|
||||
All users should migrate to the `kubo` name in their scripts and configurations.
|
||||
|
||||
#### Routing V1 HTTP API now exposed by default
|
||||
|
||||
|
||||
22
docs/changelogs/v0.40.md
Normal file
22
docs/changelogs/v0.40.md
Normal file
@ -0,0 +1,22 @@
|
||||
# Kubo changelog v0.40
|
||||
|
||||
<a href="https://ipshipyard.com/"><img align="right" src="https://github.com/user-attachments/assets/39ed3504-bb71-47f6-9bf8-cb9a1698f272" /></a>
|
||||
|
||||
This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
|
||||
|
||||
- [v0.40.0](#v0400)
|
||||
|
||||
## v0.40.0
|
||||
|
||||
- [Overview](#overview)
|
||||
- [🔦 Highlights](#-highlights)
|
||||
- [📝 Changelog](#-changelog)
|
||||
- [👨👩👧👦 Contributors](#-contributors)
|
||||
|
||||
### Overview
|
||||
|
||||
### 🔦 Highlights
|
||||
|
||||
### 📝 Changelog
|
||||
|
||||
### 👨👩👧👦 Contributors
|
||||
@ -230,6 +230,8 @@ config file at runtime.
|
||||
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
|
||||
- [`Import.UnixFSChunker`](#importunixfschunker)
|
||||
- [`Import.HashFunction`](#importhashfunction)
|
||||
- [`Import.FastProvideRoot`](#importfastprovideroot)
|
||||
- [`Import.FastProvideWait`](#importfastprovidewait)
|
||||
- [`Import.BatchMaxNodes`](#importbatchmaxnodes)
|
||||
- [`Import.BatchMaxSize`](#importbatchmaxsize)
|
||||
- [`Import.UnixFSFileMaxLinks`](#importunixfsfilemaxlinks)
|
||||
@ -1162,11 +1164,20 @@ Type: `optionalDuration`
|
||||
|
||||
### `Gateway.MaxRangeRequestFileSize`
|
||||
|
||||
Maximum file size for HTTP range requests. Range requests for files larger than this limit return 501 Not Implemented.
|
||||
Maximum file size for HTTP range requests on deserialized responses. Range requests for files larger than this limit return 501 Not Implemented.
|
||||
|
||||
Protects against CDN bugs where range requests are silently ignored and the entire file is returned instead. For example, Cloudflare's default plan returns the full file for range requests over 5GiB, causing unexpected bandwidth costs for both gateway operators and clients who only wanted a small byte range.
|
||||
**Why this exists:**
|
||||
|
||||
Set this to your CDN's range request limit (e.g., `"5GiB"` for Cloudflare's default plan). The error response suggests using verifiable block requests (application/vnd.ipld.raw) as an alternative.
|
||||
Some CDNs like Cloudflare intercept HTTP range requests and convert them to full file downloads when files exceed their cache bucket limits. Cloudflare's default plan only caches range requests for files up to 5GiB. Files larger than this receive HTTP 200 with the entire file instead of HTTP 206 with the requested byte range. A client requesting 1MB from a 40GiB file would unknowingly download all 40GiB, causing bandwidth overcharges for the gateway operator, unexpected data costs for the client, and potential browser crashes.
|
||||
|
||||
This only affects deserialized responses. Clients fetching verifiable blocks as `application/vnd.ipld.raw` are not impacted because they work with small chunks that stay well below CDN cache limits.
|
||||
|
||||
**How to use:**
|
||||
|
||||
Set this to your CDN's range request cache limit (e.g., `"5GiB"` for Cloudflare's default plan). The gateway returns 501 Not Implemented for range requests over files larger than this limit, with an error message suggesting verifiable block requests as an alternative.
|
||||
|
||||
> [!NOTE]
|
||||
> Cloudflare users running open gateway hosting deserialized responses should deploy additional protection via Cloudflare Snippets (requires Enterprise plan). The Kubo configuration alone is not sufficient because Cloudflare has already intercepted and cached the response by the time it reaches your origin. See [boxo#856](https://github.com/ipfs/boxo/issues/856#issuecomment-3523944976) for a snippet that aborts HTTP 200 responses when Content-Length exceeds the limit.
|
||||
|
||||
Default: `0` (no limit)
|
||||
|
||||
@ -2181,10 +2192,9 @@ to `false`.
|
||||
You can compare the effectiveness of sweep mode vs legacy mode by monitoring the appropriate metrics (see [Monitoring Provide Operations](#monitoring-provide-operations) above).
|
||||
|
||||
> [!NOTE]
|
||||
> This feature is opt-in for now, but will become the default in a future release.
|
||||
> Eventually, this configuration flag will be removed once the feature is stable.
|
||||
> This is the default provider system as of Kubo v0.39. To use the legacy provider instead, set `Provide.DHT.SweepEnabled=false`.
|
||||
|
||||
Default: `false`
|
||||
Default: `true`
|
||||
|
||||
Type: `flag`
|
||||
|
||||
@ -3611,6 +3621,38 @@ Default: `sha2-256`
|
||||
|
||||
Type: `optionalString`
|
||||
|
||||
### `Import.FastProvideRoot`
|
||||
|
||||
Immediately provide root CIDs to the DHT in addition to the regular provide queue.
|
||||
|
||||
This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to the `Provide.Strategy` over time. Together, they optimize for both immediate discoverability of newly imported content and efficient resource usage for complete DAG provides.
|
||||
|
||||
When disabled, only the sweep provider's queue is used.
|
||||
|
||||
This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-root` flag.
|
||||
|
||||
Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
Default: `true`
|
||||
|
||||
Type: `flag`
|
||||
|
||||
### `Import.FastProvideWait`
|
||||
|
||||
Wait for the immediate root CID provide to complete before returning.
|
||||
|
||||
When enabled, the command blocks until the provide completes, ensuring guaranteed discoverability before returning. When disabled (default), the provide happens asynchronously in the background without blocking the command.
|
||||
|
||||
Use this when you need certainty that content is discoverable before the command returns (e.g., sharing a link immediately after adding).
|
||||
|
||||
This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-wait` flag.
|
||||
|
||||
Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
Default: `false`
|
||||
|
||||
Type: `flag`
|
||||
|
||||
### `Import.BatchMaxNodes`
|
||||
|
||||
The maximum number of nodes in a write-batch. The total size of the batch is limited by `BatchMaxnodes` and `BatchMaxSize`.
|
||||
|
||||
@ -115,7 +115,7 @@ require (
|
||||
github.com/libp2p/go-doh-resolver v0.5.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.3.0 // indirect
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 // indirect
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb // indirect
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect
|
||||
github.com/libp2p/go-libp2p-pubsub v0.14.2 // indirect
|
||||
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
|
||||
@ -171,7 +171,7 @@ require (
|
||||
github.com/pion/webrtc/v4 v4.1.2 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/polydawn/refmt v0.89.0 // indirect
|
||||
github.com/probe-lab/go-libdht v0.3.0 // indirect
|
||||
github.com/probe-lab/go-libdht v0.4.0 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
|
||||
@ -430,8 +430,8 @@ github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl9
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
|
||||
github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
|
||||
github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
|
||||
@ -630,8 +630,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
|
||||
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
|
||||
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
|
||||
github.com/probe-lab/go-libdht v0.3.0 h1:Q3ZXK8wCjZvgeHSTtRrppXobXY/KHPLZJfc+cdTTyqA=
|
||||
github.com/probe-lab/go-libdht v0.3.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
|
||||
github.com/probe-lab/go-libdht v0.4.0 h1:LAqHuko/owRW6+0cs5wmJXbHzg09EUMJEh5DI37yXqo=
|
||||
github.com/probe-lab/go-libdht v0.4.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
|
||||
@ -59,7 +59,7 @@ Metrics for the legacy provider system when `Provide.DHT.SweepEnabled=false`:
|
||||
|
||||
Metrics for the DHT provider system when `Provide.DHT.SweepEnabled=true`:
|
||||
|
||||
- `total_provide_count_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`)
|
||||
- `provider_provides_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`)
|
||||
|
||||
> [!NOTE]
|
||||
> These metrics are exposed by [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/). You can enable debug logging for DHT provider activity with `GOLOG_LOG_LEVEL=dht/provider=debug`.
|
||||
|
||||
@ -47,25 +47,51 @@ Or in your IPFS config file:
|
||||
The telemetry plugin collects the following anonymized data:
|
||||
|
||||
### General Information
|
||||
- **Agent version**: The version of Kubo being used.
|
||||
- **Platform details**: Operating system, architecture, and container status.
|
||||
- **Uptime**: How long the node has been running, categorized into buckets.
|
||||
- **Repo size**: Categorized into buckets (e.g., 1GB, 5GB, 10GB, etc.).
|
||||
|
||||
- **UUID**: Anonymous identifier for this node
|
||||
- **Agent version**: Kubo version string
|
||||
- **Private network**: Whether running in a private IPFS network
|
||||
- **Repository size**: Categorized into privacy-preserving buckets (1GB, 5GB, 10GB, 100GB, 500GB, 1TB, 10TB, >10TB)
|
||||
- **Uptime**: Categorized into privacy-preserving buckets (1d, 2d, 3d, 7d, 14d, 30d, >30d)
|
||||
|
||||
### Routing & Discovery
|
||||
|
||||
- **Custom bootstrap peers**: Whether custom `Bootstrap` peers are configured
|
||||
- **Routing type**: The `Routing.Type` configured for the node
|
||||
- **Accelerated DHT client**: Whether `Routing.AcceleratedDHTClient` is enabled
|
||||
- **Delegated routing count**: Number of `Routing.DelegatedRouters` configured
|
||||
- **AutoConf enabled**: Whether `AutoConf.Enabled` is set
|
||||
- **Custom AutoConf URL**: Whether custom `AutoConf.URL` is configured
|
||||
- **mDNS**: Whether `Discovery.MDNS.Enabled` is set
|
||||
|
||||
### Content Providing
|
||||
|
||||
- **Provide and Reprovide strategy**: The `Provide.Strategy` configured
|
||||
- **Sweep-based provider**: Whether `Provide.DHT.SweepEnabled` is set
|
||||
- **Custom Interval**: Whether custom `Provide.DHT.Interval` is configured
|
||||
- **Custom MaxWorkers**: Whether custom `Provide.DHT.MaxWorkers` is configured
|
||||
|
||||
### Network Configuration
|
||||
- **Private network**: Whether the node is running in a private network.
|
||||
- **Bootstrap peers**: Whether custom bootstrap peers are used.
|
||||
- **Routing type**: Whether the node uses DHT, IPFS, or a custom routing setup.
|
||||
- **AutoNAT settings**: Whether AutoNAT is enabled and its reachability status.
|
||||
- **AutoConf settings**: Whether AutoConf is enabled and whether a custom URL is used.
|
||||
- **Swarm settings**: Whether hole punching is enabled, and whether public IP addresses are used.
|
||||
|
||||
### TLS and Discovery
|
||||
- **AutoTLS settings**: Whether WSS is enabled and whether a custom domain suffix is used.
|
||||
- **Discovery settings**: Whether mDNS is enabled.
|
||||
- **AutoNAT service mode**: The `AutoNAT.ServiceMode` configured
|
||||
- **AutoNAT reachability**: Current reachability status determined by AutoNAT
|
||||
- **Hole punching**: Whether `Swarm.EnableHolePunching` is enabled
|
||||
- **Circuit relay addresses**: Whether the node advertises circuit relay addresses
|
||||
- **Public IPv4 addresses**: Whether the node has public IPv4 addresses
|
||||
- **Public IPv6 addresses**: Whether the node has public IPv6 addresses
|
||||
- **AutoWSS**: Whether `AutoTLS.AutoWSS` is enabled
|
||||
- **Custom domain suffix**: Whether custom `AutoTLS.DomainSuffix` is configured
|
||||
|
||||
### Reprovider Strategy
|
||||
- The strategy used for reprovider (e.g., "all", "pinned"...).
|
||||
### Platform Information
|
||||
|
||||
- **Operating system**: The OS the node is running on
|
||||
- **CPU architecture**: The architecture the node is running on
|
||||
- **Container detection**: Whether the node is running inside a container
|
||||
- **VM detection**: Whether the node is running inside a virtual machine
|
||||
|
||||
### Code Reference
|
||||
|
||||
Data is organized in the `LogEvent` struct at [`plugin/plugins/telemetry/telemetry.go`](https://github.com/ipfs/kubo/blob/master/plugin/plugins/telemetry/telemetry.go). This struct is the authoritative source of truth for all telemetry data, including privacy-preserving buckets for repository size and uptime. Note that this documentation may not always be up-to-date - refer to the code for the current implementation.
|
||||
|
||||
---
|
||||
|
||||
|
||||
4
go.mod
4
go.mod
@ -53,7 +53,7 @@ require (
|
||||
github.com/libp2p/go-doh-resolver v0.5.0
|
||||
github.com/libp2p/go-libp2p v0.45.0
|
||||
github.com/libp2p/go-libp2p-http v0.5.0
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0
|
||||
github.com/libp2p/go-libp2p-pubsub v0.14.2
|
||||
github.com/libp2p/go-libp2p-pubsub-router v0.6.0
|
||||
@ -69,7 +69,7 @@ require (
|
||||
github.com/multiformats/go-multihash v0.2.3
|
||||
github.com/opentracing/opentracing-go v1.2.0
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
|
||||
github.com/probe-lab/go-libdht v0.3.0
|
||||
github.com/probe-lab/go-libdht v0.4.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
|
||||
|
||||
8
go.sum
8
go.sum
@ -514,8 +514,8 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk
|
||||
github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA=
|
||||
github.com/libp2p/go-libp2p-http v0.5.0 h1:+x0AbLaUuLBArHubbbNRTsgWz0RjNTy6DJLOxQ3/QBc=
|
||||
github.com/libp2p/go-libp2p-http v0.5.0/go.mod h1:glh87nZ35XCQyFsdzZps6+F4HYI6DctVFY5u1fehwSg=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
|
||||
@ -732,8 +732,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
|
||||
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
|
||||
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
|
||||
github.com/probe-lab/go-libdht v0.3.0 h1:Q3ZXK8wCjZvgeHSTtRrppXobXY/KHPLZJfc+cdTTyqA=
|
||||
github.com/probe-lab/go-libdht v0.3.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
|
||||
github.com/probe-lab/go-libdht v0.4.0 h1:LAqHuko/owRW6+0cs5wmJXbHzg09EUMJEh5DI37yXqo=
|
||||
github.com/probe-lab/go-libdht v0.4.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
|
||||
@ -78,6 +78,7 @@ var uptimeBuckets = []time.Duration{
|
||||
}
|
||||
|
||||
// A LogEvent is the object sent to the telemetry endpoint.
|
||||
// See https://github.com/ipfs/kubo/blob/master/docs/telemetry.md for details.
|
||||
type LogEvent struct {
|
||||
UUID string `json:"uuid"`
|
||||
|
||||
@ -91,7 +92,10 @@ type LogEvent struct {
|
||||
|
||||
UptimeBucket time.Duration `json:"uptime_bucket"`
|
||||
|
||||
ReproviderStrategy string `json:"reprovider_strategy"`
|
||||
ReproviderStrategy string `json:"reprovider_strategy"`
|
||||
ProvideDHTSweepEnabled bool `json:"provide_dht_sweep_enabled"`
|
||||
ProvideDHTIntervalCustom bool `json:"provide_dht_interval_custom"`
|
||||
ProvideDHTMaxWorkersCustom bool `json:"provide_dht_max_workers_custom"`
|
||||
|
||||
RoutingType string `json:"routing_type"`
|
||||
RoutingAcceleratedDHTClient bool `json:"routing_accelerated_dht_client"`
|
||||
@ -352,6 +356,7 @@ func (p *telemetryPlugin) Start(n *core.IpfsNode) error {
|
||||
func (p *telemetryPlugin) prepareEvent() {
|
||||
p.collectBasicInfo()
|
||||
p.collectRoutingInfo()
|
||||
p.collectProvideInfo()
|
||||
p.collectAutoNATInfo()
|
||||
p.collectAutoConfInfo()
|
||||
p.collectSwarmInfo()
|
||||
@ -360,13 +365,6 @@ func (p *telemetryPlugin) prepareEvent() {
|
||||
p.collectPlatformInfo()
|
||||
}
|
||||
|
||||
// Collects:
|
||||
// * AgentVersion
|
||||
// * PrivateNetwork
|
||||
// * RepoSizeBucket
|
||||
// * BootstrappersCustom
|
||||
// * UptimeBucket
|
||||
// * ReproviderStrategy
|
||||
func (p *telemetryPlugin) collectBasicInfo() {
|
||||
p.event.AgentVersion = ipfs.GetUserAgentVersion()
|
||||
|
||||
@ -406,8 +404,6 @@ func (p *telemetryPlugin) collectBasicInfo() {
|
||||
break
|
||||
}
|
||||
p.event.UptimeBucket = uptimeBucket
|
||||
|
||||
p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
|
||||
}
|
||||
|
||||
func (p *telemetryPlugin) collectRoutingInfo() {
|
||||
@ -416,6 +412,13 @@ func (p *telemetryPlugin) collectRoutingInfo() {
|
||||
p.event.RoutingDelegatedCount = len(p.config.Routing.DelegatedRouters)
|
||||
}
|
||||
|
||||
func (p *telemetryPlugin) collectProvideInfo() {
|
||||
p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
|
||||
p.event.ProvideDHTSweepEnabled = p.config.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled)
|
||||
p.event.ProvideDHTIntervalCustom = !p.config.Provide.DHT.Interval.IsDefault()
|
||||
p.event.ProvideDHTMaxWorkersCustom = !p.config.Provide.DHT.MaxWorkers.IsDefault()
|
||||
}
|
||||
|
||||
type reachabilityHost interface {
|
||||
Reachability() network.Reachability
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/ipfs/kubo/config"
|
||||
@ -15,6 +16,19 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// waitForLogMessage polls a buffer for a log message, waiting up to timeout duration.
|
||||
// Returns true if message found, false if timeout reached.
|
||||
func waitForLogMessage(buffer *harness.Buffer, message string, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if strings.Contains(buffer.String(), message) {
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestAdd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@ -435,7 +449,182 @@ func TestAdd(t *testing.T) {
|
||||
require.Equal(t, 992, len(root.Links))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestAddFastProvide(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
shortString = "hello world"
|
||||
shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256
|
||||
)
|
||||
|
||||
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
// Start daemon with debug logging
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
cidStr := node.IPFSAddStr(shortString)
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
// Verify fast-provide-root was disabled
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
// Use default config (FastProvideRoot=true, FastProvideWait=false)
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
cidStr := node.IPFSAddStr(shortString)
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Should see async mode started
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
|
||||
// Wait for async completion or failure (up to 11 seconds - slightly more than fastProvideTimeout)
|
||||
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
|
||||
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", 11*time.Second) ||
|
||||
waitForLogMessage(daemonLog, "async provide failed", 11*time.Second)
|
||||
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Use Runner.Run with stdin to allow for expected errors
|
||||
res := node.Runner.Run(harness.RunRequest{
|
||||
Path: node.IPFSBin,
|
||||
Args: []string{"add", "-q"},
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithStdin(strings.NewReader(shortString)),
|
||||
},
|
||||
})
|
||||
|
||||
// In sync mode (wait=true), provide errors propagate and fail the command.
|
||||
// Test environment uses 'test' profile with no bootstrappers, and CI has
|
||||
// insufficient peers for proper DHT puts, so we expect this to fail with
|
||||
// "failed to find any peer in table" error from the DHT.
|
||||
require.Equal(t, 1, res.ExitCode())
|
||||
require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table")
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Should see sync mode started
|
||||
require.Contains(t, daemonLog, "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
|
||||
require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged
|
||||
})
|
||||
|
||||
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
cidStr := node.IPFSAddStr(shortString)
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
require.Contains(t, daemonLog, "wait-flag-ignored")
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=true")
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Flag should enable it despite config saying false
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=false")
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Flag should disable it despite config saying true
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
}
|
||||
|
||||
// createDirectoryForHAMT aims to create enough files with long names for the directory block to be close to the UnixFSHAMTDirectorySizeThreshold.
|
||||
|
||||
164
test/cli/config_secrets_test.go
Normal file
164
test/cli/config_secrets_test.go
Normal file
@ -0,0 +1,164 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tidwall/sjson"
|
||||
)
|
||||
|
||||
func TestConfigSecrets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("Identity.PrivKey protection", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("Identity.PrivKey is concealed in config show", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Read the actual config file to get the real PrivKey
|
||||
configFile := node.ReadFile(node.ConfigFile())
|
||||
assert.Contains(t, configFile, "PrivKey")
|
||||
|
||||
// config show should NOT contain the PrivKey
|
||||
configShow := node.RunIPFS("config", "show").Stdout.String()
|
||||
assert.NotContains(t, configShow, "PrivKey")
|
||||
})
|
||||
|
||||
t.Run("Identity.PrivKey cannot be read via ipfs config", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Attempting to read Identity.PrivKey should fail
|
||||
res := node.RunIPFS("config", "Identity.PrivKey")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "cannot show or change private key")
|
||||
})
|
||||
|
||||
t.Run("Identity.PrivKey cannot be read via ipfs config Identity", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Attempting to read Identity section should fail (it contains PrivKey)
|
||||
res := node.RunIPFS("config", "Identity")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "cannot show or change private key")
|
||||
})
|
||||
|
||||
t.Run("Identity.PrivKey cannot be set via config replace", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Key rotation must be done in offline mode via the dedicated `ipfs key rotate` command.
|
||||
// This test ensures PrivKey cannot be changed via config replace.
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
configShow := node.RunIPFS("config", "show").Stdout.String()
|
||||
|
||||
// Try to inject a PrivKey via config replace
|
||||
configJSON := MustVal(sjson.Set(configShow, "Identity.PrivKey", "CAASqAkwggSkAgEAAo"))
|
||||
node.WriteBytes("new-config", []byte(configJSON))
|
||||
res := node.RunIPFS("config", "replace", "new-config")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "setting private key")
|
||||
})
|
||||
|
||||
t.Run("Identity.PrivKey is preserved when re-injecting config", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Read the original config file
|
||||
originalConfig := node.ReadFile(node.ConfigFile())
|
||||
assert.Contains(t, originalConfig, "PrivKey")
|
||||
|
||||
// Extract the PrivKey value for comparison
|
||||
var origPrivKey string
|
||||
assert.Contains(t, originalConfig, "PrivKey")
|
||||
// Simple extraction - find the PrivKey line
|
||||
for _, line := range strings.Split(originalConfig, "\n") {
|
||||
if strings.Contains(line, "\"PrivKey\":") {
|
||||
origPrivKey = line
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotEmpty(t, origPrivKey)
|
||||
|
||||
// Get config show output (which should NOT contain PrivKey)
|
||||
configShow := node.RunIPFS("config", "show").Stdout.String()
|
||||
assert.NotContains(t, configShow, "PrivKey")
|
||||
|
||||
// Re-inject the config via config replace
|
||||
node.WriteBytes("config-show", []byte(configShow))
|
||||
node.IPFS("config", "replace", "config-show")
|
||||
|
||||
// The PrivKey should still be in the config file
|
||||
newConfig := node.ReadFile(node.ConfigFile())
|
||||
assert.Contains(t, newConfig, "PrivKey")
|
||||
|
||||
// Verify the PrivKey line is the same
|
||||
var newPrivKey string
|
||||
for _, line := range strings.Split(newConfig, "\n") {
|
||||
if strings.Contains(line, "\"PrivKey\":") {
|
||||
newPrivKey = line
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.Equal(t, origPrivKey, newPrivKey, "PrivKey should be preserved")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("TLS security validation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("AutoConf.TLSInsecureSkipVerify defaults to false", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Check the default value in a fresh init
|
||||
res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify")
|
||||
// Field may not exist (exit code 1) or be false/empty (exit code 0)
|
||||
// Both are acceptable as they mean "not true"
|
||||
output := res.Stdout.String()
|
||||
assert.NotContains(t, output, "true", "default should not be true")
|
||||
})
|
||||
|
||||
t.Run("AutoConf.TLSInsecureSkipVerify can be set to true", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set to true
|
||||
node.IPFS("config", "AutoConf.TLSInsecureSkipVerify", "true", "--json")
|
||||
|
||||
// Verify it was set
|
||||
res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify")
|
||||
assert.Equal(t, 0, res.ExitCode())
|
||||
assert.Contains(t, res.Stdout.String(), "true")
|
||||
})
|
||||
|
||||
t.Run("HTTPRetrieval.TLSInsecureSkipVerify defaults to false", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Check the default value in a fresh init
|
||||
res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify")
|
||||
// Field may not exist (exit code 1) or be false/empty (exit code 0)
|
||||
// Both are acceptable as they mean "not true"
|
||||
output := res.Stdout.String()
|
||||
assert.NotContains(t, output, "true", "default should not be true")
|
||||
})
|
||||
|
||||
t.Run("HTTPRetrieval.TLSInsecureSkipVerify can be set to true", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set to true
|
||||
node.IPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify", "true", "--json")
|
||||
|
||||
// Verify it was set
|
||||
res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify")
|
||||
assert.Equal(t, 0, res.ExitCode())
|
||||
assert.Contains(t, res.Stdout.String(), "true")
|
||||
})
|
||||
})
|
||||
}
|
||||
@ -5,10 +5,13 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -102,3 +105,200 @@ func TestDag(t *testing.T) {
|
||||
assert.Equal(t, content, stat.Stdout.Bytes())
|
||||
})
|
||||
}
|
||||
|
||||
func TestDagImportFastProvide(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
// Start daemon with debug logging
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify fast-provide-root was disabled
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
// Use default config (FastProvideRoot=true, FastProvideWait=false)
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Should see async mode started
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
|
||||
|
||||
// Wait for async completion or failure (slightly more than DefaultFastProvideTimeout)
|
||||
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
|
||||
timeout := config.DefaultFastProvideTimeout + time.Second
|
||||
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", timeout) ||
|
||||
waitForLogMessage(daemonLog, "async provide failed", timeout)
|
||||
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file - use Run instead of IPFSDagImport to handle expected error
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
res := node.Runner.Run(harness.RunRequest{
|
||||
Path: node.IPFSBin,
|
||||
Args: []string{"dag", "import", "--pin-roots=false"},
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithStdin(r),
|
||||
},
|
||||
})
|
||||
// In sync mode (wait=true), provide errors propagate and fail the command.
|
||||
// Test environment uses 'test' profile with no bootstrappers, and CI has
|
||||
// insufficient peers for proper DHT puts, so we expect this to fail with
|
||||
// "failed to find any peer in table" error from the DHT.
|
||||
require.Equal(t, 1, res.ExitCode())
|
||||
require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table")
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Should see sync mode started
|
||||
require.Contains(t, daemonLog, "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
|
||||
require.Contains(t, daemonLog, fixtureCid) // Should log the specific CID being provided
|
||||
require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged
|
||||
})
|
||||
|
||||
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
// Note: dag import doesn't log wait-flag-ignored like add does
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file with flag override
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=true")
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Flag should enable it despite config saying false
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file with flag override
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=false")
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Flag should disable it despite config saying true
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
}
|
||||
|
||||
@ -72,9 +72,9 @@ func TestRoutingV1Proxy(t *testing.T) {
|
||||
|
||||
cidStr := nodes[0].IPFSAddStr(string(random.Bytes(1000)))
|
||||
// Reprovide as initialProviderDelay still ongoing
|
||||
res := nodes[0].IPFS("routing", "reprovide")
|
||||
require.NoError(t, res.Err)
|
||||
res = nodes[1].IPFS("routing", "findprovs", cidStr)
|
||||
waitUntilProvidesComplete(t, nodes[0])
|
||||
|
||||
res := nodes[1].IPFS("routing", "findprovs", cidStr)
|
||||
assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed())
|
||||
})
|
||||
|
||||
|
||||
@ -18,7 +18,6 @@ import (
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// swarmPeersOutput is used to parse the JSON output of 'ipfs swarm peers --enc=json'
|
||||
@ -48,9 +47,7 @@ func TestRoutingV1Server(t *testing.T) {
|
||||
text := "hello world " + uuid.New().String()
|
||||
cidStr := nodes[2].IPFSAddStr(text)
|
||||
_ = nodes[3].IPFSAddStr(text)
|
||||
// Reprovide as initialProviderDelay still ongoing
|
||||
res := nodes[3].IPFS("routing", "reprovide")
|
||||
require.NoError(t, res.Err)
|
||||
waitUntilProvidesComplete(t, nodes[3])
|
||||
|
||||
cid, err := cid.Decode(cidStr)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@ -17,6 +17,8 @@ func TestDHTOptimisticProvide(t *testing.T) {
|
||||
|
||||
nodes[0].UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Experimental.OptimisticProvide = true
|
||||
// Optimistic provide only works with the legacy provider.
|
||||
cfg.Provide.DHT.SweepEnabled = config.False
|
||||
})
|
||||
|
||||
nodes.StartDaemons().Connect()
|
||||
|
||||
384
test/cli/repo_verify_test.go
Normal file
384
test/cli/repo_verify_test.go
Normal file
@ -0,0 +1,384 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Well-known block file names in flatfs blockstore that should not be corrupted during testing.
|
||||
// Flatfs stores each block as a separate .data file on disk.
|
||||
const (
|
||||
// emptyFileFlatfsFilename is the flatfs filename for an empty UnixFS file block
|
||||
emptyFileFlatfsFilename = "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ"
|
||||
// emptyDirFlatfsFilename is the flatfs filename for an empty UnixFS directory block.
|
||||
// This block has special handling and may be served from memory even when corrupted on disk.
|
||||
emptyDirFlatfsFilename = "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"
|
||||
)
|
||||
|
||||
// getEligibleFlatfsBlockFiles returns flatfs block files (*.data) that are safe to corrupt in tests.
|
||||
// Filters out well-known blocks (empty file/dir) that cause test flakiness.
|
||||
//
|
||||
// Note: This helper is specific to the flatfs blockstore implementation where each block
|
||||
// is stored as a separate file on disk under blocks/*/*.data.
|
||||
func getEligibleFlatfsBlockFiles(t *testing.T, node *harness.Node) []string {
|
||||
blockFiles, err := filepath.Glob(filepath.Join(node.Dir, "blocks", "*", "*.data"))
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, blockFiles, "no flatfs block files found")
|
||||
|
||||
var eligible []string
|
||||
for _, f := range blockFiles {
|
||||
name := filepath.Base(f)
|
||||
if !strings.Contains(name, emptyFileFlatfsFilename) &&
|
||||
!strings.Contains(name, emptyDirFlatfsFilename) {
|
||||
eligible = append(eligible, f)
|
||||
}
|
||||
}
|
||||
return eligible
|
||||
}
|
||||
|
||||
// corruptRandomBlock corrupts a random block file in the flatfs blockstore.
|
||||
// Returns the path to the corrupted file.
|
||||
func corruptRandomBlock(t *testing.T, node *harness.Node) string {
|
||||
eligible := getEligibleFlatfsBlockFiles(t, node)
|
||||
require.NotEmpty(t, eligible, "no eligible blocks to corrupt")
|
||||
|
||||
toCorrupt := eligible[0]
|
||||
err := os.WriteFile(toCorrupt, []byte("corrupted data"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
return toCorrupt
|
||||
}
|
||||
|
||||
// corruptMultipleBlocks corrupts multiple block files in the flatfs blockstore.
|
||||
// Returns the paths to the corrupted files.
|
||||
func corruptMultipleBlocks(t *testing.T, node *harness.Node, count int) []string {
|
||||
eligible := getEligibleFlatfsBlockFiles(t, node)
|
||||
require.GreaterOrEqual(t, len(eligible), count, "not enough eligible blocks to corrupt")
|
||||
|
||||
var corrupted []string
|
||||
for i := 0; i < count && i < len(eligible); i++ {
|
||||
err := os.WriteFile(eligible[i], []byte(fmt.Sprintf("corrupted data %d", i)), 0644)
|
||||
require.NoError(t, err)
|
||||
corrupted = append(corrupted, eligible[i])
|
||||
}
|
||||
|
||||
return corrupted
|
||||
}
|
||||
|
||||
func TestRepoVerify(t *testing.T) {
|
||||
t.Run("healthy repo passes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.IPFS("add", "-q", "--raw-leaves=false", "-r", node.IPFSBin)
|
||||
|
||||
res := node.IPFS("repo", "verify")
|
||||
assert.Contains(t, res.Stdout.String(), "all blocks validated")
|
||||
})
|
||||
|
||||
t.Run("detects corruption", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.IPFSAddStr("test content")
|
||||
|
||||
corruptRandomBlock(t, node)
|
||||
|
||||
res := node.RunIPFS("repo", "verify")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stdout.String(), "was corrupt")
|
||||
assert.Contains(t, res.Stderr.String(), "1 blocks corrupt")
|
||||
})
|
||||
|
||||
t.Run("drop removes corrupt blocks", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
cid := node.IPFSAddStr("test content")
|
||||
|
||||
corruptRandomBlock(t, node)
|
||||
|
||||
res := node.RunIPFS("repo", "verify", "--drop")
|
||||
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
|
||||
output := res.Stdout.String()
|
||||
assert.Contains(t, output, "1 blocks corrupt")
|
||||
assert.Contains(t, output, "1 removed")
|
||||
|
||||
// Verify block is gone
|
||||
res = node.RunIPFS("block", "stat", cid)
|
||||
assert.NotEqual(t, 0, res.ExitCode())
|
||||
})
|
||||
|
||||
t.Run("heal requires online mode", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.IPFSAddStr("test content")
|
||||
|
||||
corruptRandomBlock(t, node)
|
||||
|
||||
res := node.RunIPFS("repo", "verify", "--heal")
|
||||
assert.NotEqual(t, 0, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "online mode")
|
||||
})
|
||||
|
||||
t.Run("heal repairs from network", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nodes := harness.NewT(t).NewNodes(2).Init()
|
||||
nodes.StartDaemons().Connect()
|
||||
defer nodes.StopDaemons()
|
||||
|
||||
// Add content to node 0
|
||||
cid := nodes[0].IPFSAddStr("test content for healing")
|
||||
|
||||
// Wait for it to appear on node 1
|
||||
nodes[1].IPFS("block", "get", cid)
|
||||
|
||||
// Corrupt on node 1
|
||||
corruptRandomBlock(t, nodes[1])
|
||||
|
||||
// Heal should restore from node 0
|
||||
res := nodes[1].RunIPFS("repo", "verify", "--heal")
|
||||
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
|
||||
output := res.Stdout.String()
|
||||
|
||||
// Should report corruption and healing with specific counts
|
||||
assert.Contains(t, output, "1 blocks corrupt")
|
||||
assert.Contains(t, output, "1 removed")
|
||||
assert.Contains(t, output, "1 healed")
|
||||
|
||||
// Verify block is restored
|
||||
nodes[1].IPFS("block", "stat", cid)
|
||||
})
|
||||
|
||||
t.Run("healed blocks contain correct data", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nodes := harness.NewT(t).NewNodes(2).Init()
|
||||
nodes.StartDaemons().Connect()
|
||||
defer nodes.StopDaemons()
|
||||
|
||||
// Add specific content to node 0
|
||||
testContent := "this is the exact content that should be healed correctly"
|
||||
cid := nodes[0].IPFSAddStr(testContent)
|
||||
|
||||
// Fetch to node 1 and verify the content is correct initially
|
||||
nodes[1].IPFS("block", "get", cid)
|
||||
res := nodes[1].IPFS("cat", cid)
|
||||
assert.Equal(t, testContent, res.Stdout.String())
|
||||
|
||||
// Corrupt on node 1
|
||||
corruptRandomBlock(t, nodes[1])
|
||||
|
||||
// Heal the corruption
|
||||
res = nodes[1].RunIPFS("repo", "verify", "--heal")
|
||||
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
|
||||
output := res.Stdout.String()
|
||||
assert.Contains(t, output, "1 blocks corrupt")
|
||||
assert.Contains(t, output, "1 removed")
|
||||
assert.Contains(t, output, "1 healed")
|
||||
|
||||
// Verify the healed content matches the original exactly
|
||||
res = nodes[1].IPFS("cat", cid)
|
||||
assert.Equal(t, testContent, res.Stdout.String(), "healed content should match original")
|
||||
|
||||
// Also verify via block get that the raw block data is correct
|
||||
block0 := nodes[0].IPFS("block", "get", cid)
|
||||
block1 := nodes[1].IPFS("block", "get", cid)
|
||||
assert.Equal(t, block0.Stdout.String(), block1.Stdout.String(), "raw block data should match")
|
||||
})
|
||||
|
||||
t.Run("multiple corrupt blocks", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Create 20 blocks
|
||||
for i := 0; i < 20; i++ {
|
||||
node.IPFSAddStr(strings.Repeat("test content ", i+1))
|
||||
}
|
||||
|
||||
// Corrupt 5 blocks
|
||||
corruptMultipleBlocks(t, node, 5)
|
||||
|
||||
// Verify detects all corruptions
|
||||
res := node.RunIPFS("repo", "verify")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
// Error summary is in stderr
|
||||
assert.Contains(t, res.Stderr.String(), "5 blocks corrupt")
|
||||
|
||||
// Test with --drop
|
||||
res = node.RunIPFS("repo", "verify", "--drop")
|
||||
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
|
||||
assert.Contains(t, res.Stdout.String(), "5 blocks corrupt")
|
||||
assert.Contains(t, res.Stdout.String(), "5 removed")
|
||||
})
|
||||
|
||||
t.Run("empty repository", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Verify empty repo passes
|
||||
res := node.IPFS("repo", "verify")
|
||||
assert.Equal(t, 0, res.ExitCode())
|
||||
assert.Contains(t, res.Stdout.String(), "all blocks validated")
|
||||
|
||||
// Should work with --drop and --heal too
|
||||
res = node.IPFS("repo", "verify", "--drop")
|
||||
assert.Equal(t, 0, res.ExitCode())
|
||||
assert.Contains(t, res.Stdout.String(), "all blocks validated")
|
||||
})
|
||||
|
||||
t.Run("partial heal success", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nodes := harness.NewT(t).NewNodes(2).Init()
|
||||
|
||||
// Start both nodes and connect them
|
||||
nodes.StartDaemons().Connect()
|
||||
defer nodes.StopDaemons()
|
||||
|
||||
// Add 5 blocks to node 0, pin them to keep available
|
||||
cid1 := nodes[0].IPFSAddStr("content available for healing 1")
|
||||
cid2 := nodes[0].IPFSAddStr("content available for healing 2")
|
||||
cid3 := nodes[0].IPFSAddStr("content available for healing 3")
|
||||
cid4 := nodes[0].IPFSAddStr("content available for healing 4")
|
||||
cid5 := nodes[0].IPFSAddStr("content available for healing 5")
|
||||
|
||||
// Pin these on node 0 to ensure they stay available
|
||||
nodes[0].IPFS("pin", "add", cid1)
|
||||
nodes[0].IPFS("pin", "add", cid2)
|
||||
nodes[0].IPFS("pin", "add", cid3)
|
||||
nodes[0].IPFS("pin", "add", cid4)
|
||||
nodes[0].IPFS("pin", "add", cid5)
|
||||
|
||||
// Node 1 fetches these blocks
|
||||
nodes[1].IPFS("block", "get", cid1)
|
||||
nodes[1].IPFS("block", "get", cid2)
|
||||
nodes[1].IPFS("block", "get", cid3)
|
||||
nodes[1].IPFS("block", "get", cid4)
|
||||
nodes[1].IPFS("block", "get", cid5)
|
||||
|
||||
// Now remove some blocks from node 0 to simulate partial availability
|
||||
nodes[0].IPFS("pin", "rm", cid3)
|
||||
nodes[0].IPFS("pin", "rm", cid4)
|
||||
nodes[0].IPFS("pin", "rm", cid5)
|
||||
nodes[0].IPFS("repo", "gc")
|
||||
|
||||
// Verify node 1 is still connected
|
||||
peers := nodes[1].IPFS("swarm", "peers")
|
||||
require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
|
||||
|
||||
// Corrupt 5 blocks on node 1
|
||||
corruptMultipleBlocks(t, nodes[1], 5)
|
||||
|
||||
// Heal should partially succeed (only cid1 and cid2 available from node 0)
|
||||
res := nodes[1].RunIPFS("repo", "verify", "--heal")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
|
||||
// Should show mixed results with specific counts in stderr
|
||||
errOutput := res.Stderr.String()
|
||||
assert.Contains(t, errOutput, "5 blocks corrupt")
|
||||
assert.Contains(t, errOutput, "5 removed")
|
||||
// Only cid1 and cid2 are available for healing, cid3-5 were GC'd
|
||||
assert.Contains(t, errOutput, "2 healed")
|
||||
assert.Contains(t, errOutput, "3 failed to heal")
|
||||
})
|
||||
|
||||
t.Run("heal with block not available on network", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nodes := harness.NewT(t).NewNodes(2).Init()
|
||||
|
||||
// Start both nodes and connect
|
||||
nodes.StartDaemons().Connect()
|
||||
defer nodes.StopDaemons()
|
||||
|
||||
// Add unique content only to node 1
|
||||
nodes[1].IPFSAddStr("unique content that exists nowhere else")
|
||||
|
||||
// Ensure nodes are connected
|
||||
peers := nodes[1].IPFS("swarm", "peers")
|
||||
require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
|
||||
|
||||
// Corrupt the block on node 1
|
||||
corruptRandomBlock(t, nodes[1])
|
||||
|
||||
// Heal should fail - node 0 doesn't have this content
|
||||
res := nodes[1].RunIPFS("repo", "verify", "--heal")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
|
||||
// Should report heal failure with specific counts in stderr
|
||||
errOutput := res.Stderr.String()
|
||||
assert.Contains(t, errOutput, "1 blocks corrupt")
|
||||
assert.Contains(t, errOutput, "1 removed")
|
||||
assert.Contains(t, errOutput, "1 failed to heal")
|
||||
})
|
||||
|
||||
t.Run("large repository scale test", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Create 1000 small blocks
|
||||
for i := 0; i < 1000; i++ {
|
||||
node.IPFSAddStr(fmt.Sprintf("content-%d", i))
|
||||
}
|
||||
|
||||
// Corrupt 10 blocks
|
||||
corruptMultipleBlocks(t, node, 10)
|
||||
|
||||
// Verify handles large repos efficiently
|
||||
res := node.RunIPFS("repo", "verify")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
|
||||
// Should report exactly 10 corrupt blocks in stderr
|
||||
assert.Contains(t, res.Stderr.String(), "10 blocks corrupt")
|
||||
|
||||
// Test --drop at scale
|
||||
res = node.RunIPFS("repo", "verify", "--drop")
|
||||
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
|
||||
output := res.Stdout.String()
|
||||
assert.Contains(t, output, "10 blocks corrupt")
|
||||
assert.Contains(t, output, "10 removed")
|
||||
})
|
||||
|
||||
t.Run("drop with partial removal failures", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Create several blocks
|
||||
for i := 0; i < 5; i++ {
|
||||
node.IPFSAddStr(fmt.Sprintf("content for removal test %d", i))
|
||||
}
|
||||
|
||||
// Corrupt 3 blocks
|
||||
corruptedFiles := corruptMultipleBlocks(t, node, 3)
|
||||
require.Len(t, corruptedFiles, 3)
|
||||
|
||||
// Make one of the corrupted files read-only to simulate removal failure
|
||||
err := os.Chmod(corruptedFiles[0], 0400) // read-only
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = os.Chmod(corruptedFiles[0], 0644) }() // cleanup
|
||||
|
||||
// Also make the directory read-only to prevent deletion
|
||||
blockDir := filepath.Dir(corruptedFiles[0])
|
||||
originalPerm, err := os.Stat(blockDir)
|
||||
require.NoError(t, err)
|
||||
err = os.Chmod(blockDir, 0500) // read+execute only, no write
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = os.Chmod(blockDir, originalPerm.Mode()) }() // cleanup
|
||||
|
||||
// Try to drop - should fail because at least one block can't be removed
|
||||
res := node.RunIPFS("repo", "verify", "--drop")
|
||||
assert.Equal(t, 1, res.ExitCode(), "should exit 1 when some blocks fail to remove")
|
||||
|
||||
// Restore permissions for verification
|
||||
_ = os.Chmod(blockDir, originalPerm.Mode())
|
||||
_ = os.Chmod(corruptedFiles[0], 0644)
|
||||
|
||||
// Should report both successes and failures with specific counts
|
||||
errOutput := res.Stderr.String()
|
||||
assert.Contains(t, errOutput, "3 blocks corrupt")
|
||||
assert.Contains(t, errOutput, "2 removed")
|
||||
assert.Contains(t, errOutput, "1 failed to remove")
|
||||
})
|
||||
}
|
||||
@ -2,7 +2,10 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
@ -10,6 +13,33 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func waitUntilProvidesComplete(t *testing.T, n *harness.Node) {
|
||||
getCidsCount := func(line string) int {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
countStr := strings.SplitN(trimmed, " ", 2)[0]
|
||||
count, err := strconv.Atoi(countStr)
|
||||
require.NoError(t, err)
|
||||
return count
|
||||
}
|
||||
|
||||
queuedProvides, ongoingProvides := true, true
|
||||
for queuedProvides || ongoingProvides {
|
||||
res := n.IPFS("provide", "stat", "-a")
|
||||
require.NoError(t, res.Err)
|
||||
for _, line := range res.Stdout.Lines() {
|
||||
if trimmed, ok := strings.CutPrefix(line, " Provide queue:"); ok {
|
||||
provideQueueSize := getCidsCount(trimmed)
|
||||
queuedProvides = provideQueueSize > 0
|
||||
}
|
||||
if trimmed, ok := strings.CutPrefix(line, " Ongoing provides:"); ok {
|
||||
ongoingProvideCount := getCidsCount(trimmed)
|
||||
ongoingProvides = ongoingProvideCount > 0
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func testRoutingDHT(t *testing.T, enablePubsub bool) {
|
||||
t.Run(fmt.Sprintf("enablePubSub=%v", enablePubsub), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@ -84,10 +114,8 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
|
||||
t.Run("ipfs routing findprovs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
hash := nodes[3].IPFSAddStr("some stuff")
|
||||
// Reprovide as initialProviderDelay still ongoing
|
||||
res := nodes[3].IPFS("routing", "reprovide")
|
||||
require.NoError(t, res.Err)
|
||||
res = nodes[4].IPFS("routing", "findprovs", hash)
|
||||
waitUntilProvidesComplete(t, nodes[3])
|
||||
res := nodes[4].IPFS("routing", "findprovs", hash)
|
||||
assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed())
|
||||
})
|
||||
|
||||
|
||||
@ -159,4 +159,127 @@ func TestRPCAuth(t *testing.T) {
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
|
||||
t.Run("Requests without Authorization header are rejected when auth is enabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
|
||||
"userA": {
|
||||
AuthSecret: "bearer:mytoken",
|
||||
AllowedPaths: []string{"/api/v0"},
|
||||
},
|
||||
})
|
||||
|
||||
// Create client with NO auth
|
||||
apiClient := node.APIClient() // Uses http.DefaultClient with no auth headers
|
||||
|
||||
// Should be denied without auth header
|
||||
resp := apiClient.Post("/api/v0/id", nil)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
|
||||
// Should contain denial message
|
||||
assert.Contains(t, resp.Body, rpcDeniedMsg)
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
|
||||
t.Run("Version endpoint is always accessible even with limited AllowedPaths", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
|
||||
"userA": {
|
||||
AuthSecret: "bearer:mytoken",
|
||||
AllowedPaths: []string{"/api/v0/id"}, // Only /id allowed
|
||||
},
|
||||
})
|
||||
|
||||
apiClient := node.APIClient()
|
||||
apiClient.Client = &http.Client{
|
||||
Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport),
|
||||
}
|
||||
|
||||
// Can access /version even though not in AllowedPaths
|
||||
resp := apiClient.Post("/api/v0/version", nil)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
|
||||
t.Run("User cannot access API with another user's secret", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
|
||||
"alice": {
|
||||
AuthSecret: "bearer:alice-secret",
|
||||
AllowedPaths: []string{"/api/v0/id"},
|
||||
},
|
||||
"bob": {
|
||||
AuthSecret: "bearer:bob-secret",
|
||||
AllowedPaths: []string{"/api/v0/config"},
|
||||
},
|
||||
})
|
||||
|
||||
// Alice tries to use Bob's secret
|
||||
apiClient := node.APIClient()
|
||||
apiClient.Client = &http.Client{
|
||||
Transport: auth.NewAuthorizedRoundTripper("Bearer bob-secret", http.DefaultTransport),
|
||||
}
|
||||
|
||||
// Bob's secret should work for Bob's paths
|
||||
resp := apiClient.Post("/api/v0/config/show", nil)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
// But not for Alice's paths (Bob doesn't have access to /id)
|
||||
resp = apiClient.Post("/api/v0/id", nil)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
|
||||
t.Run("Empty AllowedPaths denies all access except version", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
|
||||
"userA": {
|
||||
AuthSecret: "bearer:mytoken",
|
||||
AllowedPaths: []string{}, // Empty!
|
||||
},
|
||||
})
|
||||
|
||||
apiClient := node.APIClient()
|
||||
apiClient.Client = &http.Client{
|
||||
Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport),
|
||||
}
|
||||
|
||||
// Should deny everything
|
||||
resp := apiClient.Post("/api/v0/id", nil)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
|
||||
resp = apiClient.Post("/api/v0/config/show", nil)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
|
||||
// Except version
|
||||
resp = apiClient.Post("/api/v0/version", nil)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
|
||||
t.Run("CLI commands fail without --api-auth when auth is enabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
|
||||
"userA": {
|
||||
AuthSecret: "bearer:mytoken",
|
||||
AllowedPaths: []string{"/api/v0"},
|
||||
},
|
||||
})
|
||||
|
||||
// Try to run command without --api-auth flag
|
||||
resp := node.RunIPFS("id") // No --api-auth flag
|
||||
require.Error(t, resp.Err)
|
||||
require.Contains(t, resp.Stderr.String(), rpcDeniedMsg)
|
||||
|
||||
node.StopDaemon()
|
||||
})
|
||||
}
|
||||
|
||||
@ -205,6 +205,9 @@ func TestTelemetry(t *testing.T) {
|
||||
"repo_size_bucket",
|
||||
"uptime_bucket",
|
||||
"reprovider_strategy",
|
||||
"provide_dht_sweep_enabled",
|
||||
"provide_dht_interval_custom",
|
||||
"provide_dht_max_workers_custom",
|
||||
"routing_type",
|
||||
"routing_accelerated_dht_client",
|
||||
"routing_delegated_count",
|
||||
|
||||
@ -184,7 +184,7 @@ require (
|
||||
github.com/libp2p/go-flow-metrics v0.3.0 // indirect
|
||||
github.com/libp2p/go-libp2p v0.45.0 // indirect
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 // indirect
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb // indirect
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect
|
||||
github.com/libp2p/go-libp2p-record v0.3.1 // indirect
|
||||
github.com/libp2p/go-libp2p-routing-helpers v0.7.6-0.20251016083611-f098f492895e // indirect
|
||||
|
||||
@ -464,8 +464,8 @@ github.com/libp2p/go-libp2p v0.45.0 h1:Pdhr2HsFXaYjtfiNcBP4CcRUONvbMFdH3puM9vV4T
|
||||
github.com/libp2p/go-libp2p v0.45.0/go.mod h1:NovCojezAt4dnDd4fH048K7PKEqH0UFYYqJRjIIu8zc=
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
|
||||
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
|
||||
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
|
||||
github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg=
|
||||
|
||||
@ -93,8 +93,8 @@ EOF
|
||||
test_cmp expected actual
|
||||
'
|
||||
|
||||
test_expect_failure "'ipfs add' with an unregistered hash and wrapped leaves fails without crashing" '
|
||||
ipfs add --hash poseidon-bls12_381-a2-fc1 --raw-leaves=false -r mountdir/planets
|
||||
test_expect_success "'ipfs add' with an unregistered hash and wrapped leaves fails without crashing" '
|
||||
test_expect_code 1 ipfs add --hash poseidon-bls12_381-a2-fc1 --raw-leaves=false -r mountdir/planets
|
||||
'
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,9 @@
|
||||
# Copyright (c) 2016 Jeromy Johnson
|
||||
# MIT Licensed; see the LICENSE file in this repository.
|
||||
#
|
||||
# NOTE: This is a legacy sharness test kept for compatibility.
|
||||
# New tests for 'ipfs repo verify' should be added to test/cli/repo_verify_test.go
|
||||
#
|
||||
|
||||
test_description="Test ipfs repo fsck"
|
||||
|
||||
|
||||
@ -250,6 +250,5 @@ process_resident_memory_bytes
|
||||
process_start_time_seconds
|
||||
process_virtual_memory_bytes
|
||||
process_virtual_memory_max_bytes
|
||||
provider_reprovider_provide_count
|
||||
provider_reprovider_reprovide_count
|
||||
provider_provides_total
|
||||
target_info
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
var CurrentCommit string
|
||||
|
||||
// CurrentVersionNumber is the current application's version literal.
|
||||
const CurrentVersionNumber = "0.39.0-dev"
|
||||
const CurrentVersionNumber = "0.40.0-dev"
|
||||
|
||||
const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user