Merge remote-tracking branch 'origin/master' into feat/get-closest-peers

This commit is contained in:
Hector Sanjuan 2025-11-19 11:32:12 +01:00
commit 0b07da9c94
50 changed files with 2717 additions and 187 deletions

View File

@ -1,5 +1,6 @@
# Kubo Changelogs
- [v0.40](docs/changelogs/v0.40.md)
- [v0.39](docs/changelogs/v0.39.md)
- [v0.38](docs/changelogs/v0.38.md)
- [v0.37](docs/changelogs/v0.37.md)

View File

@ -16,6 +16,8 @@ const (
DefaultUnixFSRawLeaves = false
DefaultUnixFSChunker = "size-262144"
DefaultHashFunction = "sha2-256"
DefaultFastProvideRoot = true
DefaultFastProvideWait = false
DefaultUnixFSHAMTDirectorySizeThreshold = 262144 // 256KiB - https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26
@ -48,6 +50,8 @@ type Import struct {
UnixFSHAMTDirectorySizeThreshold OptionalBytes
BatchMaxNodes OptionalInteger
BatchMaxSize OptionalInteger
FastProvideRoot Flag
FastProvideWait Flag
}
// ValidateImportConfig validates the Import configuration according to UnixFS spec requirements.

View File

@ -15,13 +15,18 @@ const (
// DHT provider defaults
DefaultProvideDHTInterval = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326
DefaultProvideDHTMaxWorkers = 16 // Unified default for both sweep and legacy providers
DefaultProvideDHTSweepEnabled = false
DefaultProvideDHTSweepEnabled = true
DefaultProvideDHTResumeEnabled = true
DefaultProvideDHTDedicatedPeriodicWorkers = 2
DefaultProvideDHTDedicatedBurstWorkers = 1
DefaultProvideDHTMaxProvideConnsPerWorker = 20
DefaultProvideDHTKeystoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes)
DefaultProvideDHTOfflineDelay = 2 * time.Hour
// DefaultFastProvideTimeout is the maximum time allowed for fast-provide operations.
// Prevents hanging on network issues when providing root CID.
// 10 seconds is sufficient for DHT operations with sweep provider or accelerated client.
DefaultFastProvideTimeout = 10 * time.Second
)
type ProvideStrategy int
@ -64,7 +69,7 @@ type ProvideDHT struct {
MaxWorkers *OptionalInteger `json:",omitempty"`
// SweepEnabled activates the sweeping reprovider system which spreads
// reprovide operations over time. This will become the default in a future release.
// reprovide operations over time.
// Default: DefaultProvideDHTSweepEnabled
SweepEnabled Flag `json:",omitempty"`
@ -175,3 +180,25 @@ func ValidateProvideConfig(cfg *Provide) error {
return nil
}
// ShouldProvideForStrategy determines if content should be provided based on the provide strategy
// and content characteristics (pinned status, root status, MFS status).
func ShouldProvideForStrategy(strategy ProvideStrategy, isPinned bool, isPinnedRoot bool, isMFS bool) bool {
if strategy == ProvideStrategyAll {
// 'all' strategy: always provide
return true
}
// For combined strategies, check each component
if strategy&ProvideStrategyPinned != 0 && isPinned {
return true
}
if strategy&ProvideStrategyRoots != 0 && isPinnedRoot {
return true
}
if strategy&ProvideStrategyMFS != 0 && isMFS {
return true
}
return false
}

View File

@ -105,3 +105,87 @@ func TestValidateProvideConfig_MaxWorkers(t *testing.T) {
})
}
}
func TestShouldProvideForStrategy(t *testing.T) {
t.Run("all strategy always provides", func(t *testing.T) {
// ProvideStrategyAll should return true regardless of flags
testCases := []struct{ pinned, pinnedRoot, mfs bool }{
{false, false, false},
{true, true, true},
{true, false, false},
}
for _, tc := range testCases {
assert.True(t, ShouldProvideForStrategy(
ProvideStrategyAll, tc.pinned, tc.pinnedRoot, tc.mfs))
}
})
t.Run("single strategies match only their flag", func(t *testing.T) {
tests := []struct {
name string
strategy ProvideStrategy
pinned, pinnedRoot, mfs bool
want bool
}{
{"pinned: matches when pinned=true", ProvideStrategyPinned, true, false, false, true},
{"pinned: ignores other flags", ProvideStrategyPinned, false, true, true, false},
{"roots: matches when pinnedRoot=true", ProvideStrategyRoots, false, true, false, true},
{"roots: ignores other flags", ProvideStrategyRoots, true, false, true, false},
{"mfs: matches when mfs=true", ProvideStrategyMFS, false, false, true, true},
{"mfs: ignores other flags", ProvideStrategyMFS, true, true, false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs)
assert.Equal(t, tt.want, got)
})
}
})
t.Run("combined strategies use OR logic (else-if bug fix)", func(t *testing.T) {
// CRITICAL: Tests the fix where bitflag combinations (pinned+mfs) didn't work
// because of else-if instead of separate if statements
tests := []struct {
name string
strategy ProvideStrategy
pinned, pinnedRoot, mfs bool
want bool
}{
// pinned|mfs: provide if EITHER matches
{"pinned|mfs when pinned", ProvideStrategyPinned | ProvideStrategyMFS, true, false, false, true},
{"pinned|mfs when mfs", ProvideStrategyPinned | ProvideStrategyMFS, false, false, true, true},
{"pinned|mfs when both", ProvideStrategyPinned | ProvideStrategyMFS, true, false, true, true},
{"pinned|mfs when neither", ProvideStrategyPinned | ProvideStrategyMFS, false, false, false, false},
// roots|mfs
{"roots|mfs when root", ProvideStrategyRoots | ProvideStrategyMFS, false, true, false, true},
{"roots|mfs when mfs", ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true},
{"roots|mfs when neither", ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false},
// pinned|roots
{"pinned|roots when pinned", ProvideStrategyPinned | ProvideStrategyRoots, true, false, false, true},
{"pinned|roots when root", ProvideStrategyPinned | ProvideStrategyRoots, false, true, false, true},
{"pinned|roots when neither", ProvideStrategyPinned | ProvideStrategyRoots, false, false, false, false},
// triple combination
{"all-three when any matches", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true},
{"all-three when none match", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs)
assert.Equal(t, tt.want, got)
})
}
})
t.Run("zero strategy never provides", func(t *testing.T) {
assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), false, false, false))
assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), true, true, true))
})
}

View File

@ -117,6 +117,16 @@ func (f Flag) String() string {
}
}
// ResolveBoolFromConfig returns the resolved boolean value based on:
// - If userSet is true, returns userValue (user explicitly set the flag)
// - Otherwise, uses configFlag.WithDefault(defaultValue) (respects config or falls back to default)
func ResolveBoolFromConfig(userValue bool, userSet bool, configFlag Flag, defaultValue bool) bool {
if userSet {
return userValue
}
return configFlag.WithDefault(defaultValue)
}
var (
_ json.Unmarshaler = (*Flag)(nil)
_ json.Marshaler = (*Flag)(nil)

View File

@ -61,20 +61,45 @@ const (
inlineLimitOptionName = "inline-limit"
toFilesOptionName = "to-files"
preserveModeOptionName = "preserve-mode"
preserveMtimeOptionName = "preserve-mtime"
modeOptionName = "mode"
mtimeOptionName = "mtime"
mtimeNsecsOptionName = "mtime-nsecs"
preserveModeOptionName = "preserve-mode"
preserveMtimeOptionName = "preserve-mtime"
modeOptionName = "mode"
mtimeOptionName = "mtime"
mtimeNsecsOptionName = "mtime-nsecs"
fastProvideRootOptionName = "fast-provide-root"
fastProvideWaitOptionName = "fast-provide-wait"
)
const adderOutChanSize = 8
const (
adderOutChanSize = 8
)
var AddCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Add a file or directory to IPFS.",
ShortDescription: `
Adds the content of <path> to IPFS. Use -r to add directories (recursively).
FAST PROVIDE OPTIMIZATION:
When you add content to IPFS, the sweep provider queues it for efficient
DHT provides over time. While this is resource-efficient, other peers won't
find your content immediately after 'ipfs add' completes.
To make sharing faster, 'ipfs add' does an immediate provide of the root CID
to the DHT in addition to the regular queue. This complements the sweep provider:
fast-provide handles the urgent case (root CIDs that users share and reference),
while the sweep provider efficiently provides all blocks according to
Provide.Strategy over time.
By default, this immediate provide runs in the background without blocking
the command. If you need certainty that the root CID is discoverable before
the command returns (e.g., sharing a link immediately), use --fast-provide-wait
to wait for the provide to complete. Use --fast-provide-root=false to skip
this optimization.
This works best with the sweep provider and accelerated DHT client.
Automatically skipped when DHT is not available.
`,
LongDescription: `
Adds the content of <path> to IPFS. Use -r to add directories.
@ -213,6 +238,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"),
cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). WARNING: experimental, forces dag-pb for root block, disables raw-leaves"),
cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CID to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"),
},
PreRun: func(req *cmds.Request, env cmds.Environment) error {
quiet, _ := req.Options[quietOptionName].(bool)
@ -283,6 +310,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
mode, _ := req.Options[modeOptionName].(uint)
mtime, _ := req.Options[mtimeOptionName].(int64)
mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
if chunker == "" {
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
@ -319,6 +348,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout))
}
fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot)
fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait)
// Storing optional mode or mtime (UnixFS 1.5) requires root block
// to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible.
if preserveMode || preserveMtime || mode != 0 || mtime != 0 {
@ -421,11 +453,12 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
}
var added int
var fileAddedToMFS bool
var lastRootCid path.ImmutablePath // Track the root CID for fast-provide
addit := toadd.Entries()
for addit.Next() {
_, dir := addit.Node().(files.Directory)
errCh := make(chan error, 1)
events := make(chan interface{}, adderOutChanSize)
events := make(chan any, adderOutChanSize)
opts[len(opts)-1] = options.Unixfs.Events(events)
go func() {
@ -437,6 +470,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
return
}
// Store the root CID for potential fast-provide operation
lastRootCid = pathAdded
// creating MFS pointers when optional --to-files is set
if toFilesSet {
if addit.Name() == "" {
@ -560,12 +596,29 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
return fmt.Errorf("expected a file argument")
}
// Apply fast-provide-root if the flag is enabled
if fastProvideRoot && (lastRootCid != path.ImmutablePath{}) {
cfg, err := ipfsNode.Repo.Config()
if err != nil {
return err
}
if err := cmdenv.ExecuteFastProvide(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet); err != nil {
return err
}
} else if !fastProvideRoot {
if fastProvideWait {
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
} else {
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
}
}
return nil
},
PostRun: cmds.PostRunMap{
cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
sizeChan := make(chan int64, 1)
outChan := make(chan interface{})
outChan := make(chan any)
req := res.Request()
// Could be slow.

View File

@ -1,15 +1,19 @@
package cmdenv
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/ipfs/kubo/commands"
"github.com/ipfs/kubo/core"
"github.com/ipfs/go-cid"
cmds "github.com/ipfs/go-ipfs-cmds"
logging "github.com/ipfs/go-log/v2"
routing "github.com/libp2p/go-libp2p/core/routing"
"github.com/ipfs/kubo/commands"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core"
coreiface "github.com/ipfs/kubo/core/coreiface"
options "github.com/ipfs/kubo/core/coreiface/options"
)
@ -86,3 +90,103 @@ func needEscape(s string) bool {
}
return false
}
// provideCIDSync performs a synchronous/blocking provide operation to announce
// the given CID to the DHT.
//
// - If the accelerated DHT client is used, a DHT lookup isn't needed, we
// directly allocate provider records to closest peers.
// - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an
// optimistic provide call.
// - Else we make a standard provide call (much slower).
//
// IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient()
// before calling this function. Calling with a nil or invalid router will cause
// a panic - this is the caller's responsibility to prevent.
func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error {
return router.Provide(ctx, c, true)
}
// ExecuteFastProvide immediately provides a root CID to the DHT, bypassing the regular
// provide queue for faster content discovery. This function is reusable across commands
// that add or import content, such as ipfs add and ipfs dag import.
//
// Parameters:
// - ctx: context for synchronous provides
// - ipfsNode: the IPFS node instance
// - cfg: node configuration
// - rootCid: the CID to provide
// - wait: whether to block until provide completes (sync mode)
// - isPinned: whether content is pinned
// - isPinnedRoot: whether this is a pinned root CID
// - isMFS: whether content is in MFS
//
// Return value:
// - Returns nil if operation succeeded or was skipped (preconditions not met)
// - Returns error only in sync mode (wait=true) when provide operation fails
// - In async mode (wait=false), always returns nil (errors logged in goroutine)
//
// The function handles all precondition checks (Provide.Enabled, DHT availability,
// strategy matching) and logs appropriately. In async mode, it launches a goroutine
// with a detached context and timeout.
func ExecuteFastProvide(
ctx context.Context,
ipfsNode *core.IpfsNode,
cfg *config.Config,
rootCid cid.Cid,
wait bool,
isPinned bool,
isPinnedRoot bool,
isMFS bool,
) error {
log.Debugw("fast-provide-root: enabled", "wait", wait)
// Check preconditions for providing
switch {
case !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled):
log.Debugw("fast-provide-root: skipped", "reason", "Provide.Enabled is false")
return nil
case cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0:
log.Debugw("fast-provide-root: skipped", "reason", "Provide.DHT.Interval is 0")
return nil
case !ipfsNode.HasActiveDHTClient():
log.Debugw("fast-provide-root: skipped", "reason", "DHT not available")
return nil
}
// Check if strategy allows providing this content
strategyStr := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
strategy := config.ParseProvideStrategy(strategyStr)
shouldProvide := config.ShouldProvideForStrategy(strategy, isPinned, isPinnedRoot, isMFS)
if !shouldProvide {
log.Debugw("fast-provide-root: skipped", "reason", "strategy does not match content", "strategy", strategyStr, "pinned", isPinned, "pinnedRoot", isPinnedRoot, "mfs", isMFS)
return nil
}
// Execute provide operation
if wait {
// Synchronous mode: block until provide completes, return error on failure
log.Debugw("fast-provide-root: providing synchronously", "cid", rootCid)
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
log.Warnw("fast-provide-root: sync provide failed", "cid", rootCid, "error", err)
return fmt.Errorf("fast-provide: %w", err)
}
log.Debugw("fast-provide-root: sync provide completed", "cid", rootCid)
return nil
}
// Asynchronous mode (default): fire-and-forget, don't block, always return nil
log.Debugw("fast-provide-root: providing asynchronously", "cid", rootCid)
go func() {
// Use detached context with timeout to prevent hanging on network issues
ctx, cancel := context.WithTimeout(context.Background(), config.DefaultFastProvideTimeout)
defer cancel()
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
log.Warnw("fast-provide-root: async provide failed", "cid", rootCid, "error", err)
} else {
log.Debugw("fast-provide-root: async provide completed", "cid", rootCid)
}
}()
return nil
}

View File

@ -74,10 +74,13 @@ func PathOrCidPath(str string) (path.Path, error) {
return p, nil
}
// Save the original error before attempting fallback
originalErr := err
if p, err := path.NewPath("/ipfs/" + str); err == nil {
return p, nil
}
// Send back original err.
return nil, err
return nil, originalErr
}

View File

@ -0,0 +1,106 @@
package cmdutils
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPathOrCidPath(t *testing.T) {
t.Run("valid path is returned as-is", func(t *testing.T) {
validPath := "/ipfs/QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"
p, err := PathOrCidPath(validPath)
require.NoError(t, err)
assert.Equal(t, validPath, p.String())
})
t.Run("valid CID is converted to /ipfs/ path", func(t *testing.T) {
cid := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"
p, err := PathOrCidPath(cid)
require.NoError(t, err)
assert.Equal(t, "/ipfs/"+cid, p.String())
})
t.Run("valid ipns path is returned as-is", func(t *testing.T) {
validPath := "/ipns/example.com"
p, err := PathOrCidPath(validPath)
require.NoError(t, err)
assert.Equal(t, validPath, p.String())
})
t.Run("returns original error when both attempts fail", func(t *testing.T) {
invalidInput := "invalid!@#path"
_, err := PathOrCidPath(invalidInput)
require.Error(t, err)
// The error should reference the original input attempt.
// This ensures users get meaningful error messages about their actual input.
assert.Contains(t, err.Error(), invalidInput,
"error should mention the original input")
assert.Contains(t, err.Error(), "path does not have enough components",
"error should describe the problem with the original input")
})
t.Run("empty string returns error about original input", func(t *testing.T) {
_, err := PathOrCidPath("")
require.Error(t, err)
// Verify we're not getting an error about "/ipfs/" (the fallback)
errMsg := err.Error()
assert.NotContains(t, errMsg, "/ipfs/",
"error should be about empty input, not the fallback path")
})
t.Run("invalid characters return error about original input", func(t *testing.T) {
invalidInput := "not a valid path or CID with spaces and /@#$%"
_, err := PathOrCidPath(invalidInput)
require.Error(t, err)
// The error message should help debug the original input
assert.True(t, strings.Contains(err.Error(), invalidInput) ||
strings.Contains(err.Error(), "invalid"),
"error should reference original problematic input")
})
t.Run("CID with path is converted correctly", func(t *testing.T) {
cidWithPath := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG/file.txt"
p, err := PathOrCidPath(cidWithPath)
require.NoError(t, err)
assert.Equal(t, "/ipfs/"+cidWithPath, p.String())
})
}
func TestValidatePinName(t *testing.T) {
t.Run("valid pin name is accepted", func(t *testing.T) {
err := ValidatePinName("my-pin-name")
assert.NoError(t, err)
})
t.Run("empty pin name is accepted", func(t *testing.T) {
err := ValidatePinName("")
assert.NoError(t, err)
})
t.Run("pin name at max length is accepted", func(t *testing.T) {
maxName := strings.Repeat("a", MaxPinNameBytes)
err := ValidatePinName(maxName)
assert.NoError(t, err)
})
t.Run("pin name exceeding max length is rejected", func(t *testing.T) {
tooLong := strings.Repeat("a", MaxPinNameBytes+1)
err := ValidatePinName(tooLong)
require.Error(t, err)
assert.Contains(t, err.Error(), "max")
})
t.Run("pin name with unicode is counted by bytes", func(t *testing.T) {
// Unicode character can be multiple bytes
unicodeName := strings.Repeat("🔒", MaxPinNameBytes/4+1) // emoji is 4 bytes
err := ValidatePinName(unicodeName)
require.Error(t, err)
assert.Contains(t, err.Error(), "bytes")
})
}

View File

@ -16,10 +16,12 @@ import (
)
const (
pinRootsOptionName = "pin-roots"
progressOptionName = "progress"
silentOptionName = "silent"
statsOptionName = "stats"
pinRootsOptionName = "pin-roots"
progressOptionName = "progress"
silentOptionName = "silent"
statsOptionName = "stats"
fastProvideRootOptionName = "fast-provide-root"
fastProvideWaitOptionName = "fast-provide-wait"
)
// DagCmd provides a subset of commands for interacting with ipld dag objects
@ -189,6 +191,18 @@ Note:
currently present in the blockstore does not represent a complete DAG,
pinning of that individual root will fail.
FAST PROVIDE OPTIMIZATION:
Root CIDs from CAR headers are immediately provided to the DHT in addition
to the regular provide queue, allowing other peers to discover your content
right away. This complements the sweep provider, which efficiently provides
all blocks according to Provide.Strategy over time.
By default, the provide happens in the background without blocking the
command. Use --fast-provide-wait to wait for the provide to complete, or
--fast-provide-root=false to skip it. Works even with --pin-roots=false.
Automatically skipped when DHT is not available.
Maximum supported CAR version: 2
Specification of CAR formats: https://ipld.io/specs/transport/car/
`,
@ -200,6 +214,8 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
cmds.BoolOption(silentOptionName, "No output."),
cmds.BoolOption(statsOptionName, "Output stats."),
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"),
cmdutils.AllowBigBlockOption,
},
Type: CarImportOutput{},

View File

@ -11,6 +11,7 @@ import (
cmds "github.com/ipfs/go-ipfs-cmds"
ipld "github.com/ipfs/go-ipld-format"
ipldlegacy "github.com/ipfs/go-ipld-legacy"
logging "github.com/ipfs/go-log/v2"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/coreiface/options"
gocarv2 "github.com/ipld/go-car/v2"
@ -19,6 +20,8 @@ import (
"github.com/ipfs/kubo/core/commands/cmdutils"
)
var log = logging.Logger("core/commands")
func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
node, err := cmdenv.GetNode(env)
if err != nil {
@ -47,6 +50,12 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot)
fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait)
// grab a pinlock ( which doubles as a GC lock ) so that regardless of the
// size of the streamed-in cars nothing will disappear on us before we had
// a chance to roots that may show up at the very end
@ -191,5 +200,21 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
}
}
// Fast-provide roots for faster discovery
if fastProvideRoot {
err = roots.ForEach(func(c cid.Cid) error {
return cmdenv.ExecuteFastProvide(req.Context, node, cfg, c, fastProvideWait, doPinRoots, doPinRoots, false)
})
if err != nil {
return err
}
} else {
if fastProvideWait {
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
} else {
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
}
}
return nil
}

View File

@ -56,7 +56,7 @@ var queryDhtCmd = &cmds.Command{
return err
}
if nd.DHTClient == nil {
if !nd.HasActiveDHTClient() {
return ErrNotDHT
}
@ -70,7 +70,7 @@ var queryDhtCmd = &cmds.Command{
ctx, events := routing.RegisterForQueryEvents(ctx)
client := nd.DHTClient
if client == nd.DHT {
if nd.DHT != nil && client == nd.DHT {
client = nd.DHT.WAN
if !nd.DHT.WANActive() {
client = nd.DHT.LAN

View File

@ -1,6 +1,7 @@
package commands
import (
"context"
"errors"
"fmt"
"io"
@ -11,6 +12,7 @@ import (
humanize "github.com/dustin/go-humanize"
boxoprovider "github.com/ipfs/boxo/provider"
cid "github.com/ipfs/go-cid"
cmds "github.com/ipfs/go-ipfs-cmds"
"github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
@ -18,6 +20,7 @@ import (
"github.com/libp2p/go-libp2p-kad-dht/provider/buffered"
"github.com/libp2p/go-libp2p-kad-dht/provider/dual"
"github.com/libp2p/go-libp2p-kad-dht/provider/stats"
routing "github.com/libp2p/go-libp2p/core/routing"
"github.com/probe-lab/go-libdht/kad/key"
"golang.org/x/exp/constraints"
)
@ -575,3 +578,19 @@ func humanInt[T constraints.Integer](val T) string {
func humanFull(val float64, decimals int) string {
return humanize.CommafWithDigits(val, decimals)
}
// provideCIDSync performs a synchronous/blocking provide operation to announce
// the given CID to the DHT.
//
// - If the accelerated DHT client is used, a DHT lookup isn't needed, we
// directly allocate provider records to closest peers.
// - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an
// optimistic provide call.
// - Else we make a standard provide call (much slower).
//
// IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient()
// before calling this function. Calling with a nil or invalid router will cause
// a panic - this is the caller's responsibility to prevent.
func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error {
return router.Provide(ctx, c, true)
}

View File

@ -5,20 +5,22 @@ import (
"errors"
"fmt"
"io"
"os"
"runtime"
"strings"
"sync"
"text/tabwriter"
"time"
oldcmds "github.com/ipfs/kubo/commands"
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
coreiface "github.com/ipfs/kubo/core/coreiface"
corerepo "github.com/ipfs/kubo/core/corerepo"
fsrepo "github.com/ipfs/kubo/repo/fsrepo"
"github.com/ipfs/kubo/repo/fsrepo/migrations"
humanize "github.com/dustin/go-humanize"
bstore "github.com/ipfs/boxo/blockstore"
"github.com/ipfs/boxo/path"
cid "github.com/ipfs/go-cid"
cmds "github.com/ipfs/go-ipfs-cmds"
)
@ -226,45 +228,137 @@ Version string The repo version.
},
}
// VerifyProgress reports verification progress to the user.
// It contains either a message about a corrupt block or a progress counter.
type VerifyProgress struct {
Msg string
Progress int
Msg string // Message about a corrupt/healed block (empty for valid blocks)
Progress int // Number of blocks processed so far
}
func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- string, bs bstore.Blockstore) {
// verifyState represents the state of a block after verification.
// States track both the verification result and any remediation actions taken.
type verifyState int
const (
verifyStateValid verifyState = iota // Block is valid and uncorrupted
verifyStateCorrupt // Block is corrupt, no action taken
verifyStateCorruptRemoved // Block was corrupt and successfully removed
verifyStateCorruptRemoveFailed // Block was corrupt but removal failed
verifyStateCorruptHealed // Block was corrupt, removed, and successfully re-fetched
verifyStateCorruptHealFailed // Block was corrupt and removed, but re-fetching failed
)
const (
// verifyWorkerMultiplier determines worker pool size relative to CPU count.
// Since block verification is I/O-bound (disk reads + potential network fetches),
// we use more workers than CPU cores to maximize throughput.
verifyWorkerMultiplier = 2
)
// verifyResult contains the outcome of verifying a single block.
// It includes the block's CID, its verification state, and an optional
// human-readable message describing what happened.
type verifyResult struct {
cid cid.Cid // CID of the block that was verified
state verifyState // Final state after verification and any remediation
msg string // Human-readable message (empty for valid blocks)
}
// verifyWorkerRun processes CIDs from the keys channel, verifying their integrity.
// If shouldDrop is true, corrupt blocks are removed from the blockstore.
// If shouldHeal is true (implies shouldDrop), removed blocks are re-fetched from the network.
// The api parameter must be non-nil when shouldHeal is true.
// healTimeout specifies the maximum time to wait for each block heal (0 = no timeout).
func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- *verifyResult, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) {
defer wg.Done()
sendResult := func(r *verifyResult) bool {
select {
case results <- r:
return true
case <-ctx.Done():
return false
}
}
for k := range keys {
_, err := bs.Get(ctx, k)
if err != nil {
select {
case results <- fmt.Sprintf("block %s was corrupt (%s)", k, err):
case <-ctx.Done():
return
// Block is corrupt
result := &verifyResult{cid: k, state: verifyStateCorrupt}
if !shouldDrop {
result.msg = fmt.Sprintf("block %s was corrupt (%s)", k, err)
if !sendResult(result) {
return
}
continue
}
// Try to delete
if delErr := bs.DeleteBlock(ctx, k); delErr != nil {
result.state = verifyStateCorruptRemoveFailed
result.msg = fmt.Sprintf("block %s was corrupt (%s), failed to remove (%s)", k, err, delErr)
if !sendResult(result) {
return
}
continue
}
if !shouldHeal {
result.state = verifyStateCorruptRemoved
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed", k, err)
if !sendResult(result) {
return
}
continue
}
// Try to heal by re-fetching from network (api is guaranteed non-nil here)
healCtx := ctx
var healCancel context.CancelFunc
if healTimeout > 0 {
healCtx, healCancel = context.WithTimeout(ctx, healTimeout)
}
if _, healErr := api.Block().Get(healCtx, path.FromCid(k)); healErr != nil {
result.state = verifyStateCorruptHealFailed
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, failed to heal (%s)", k, err, healErr)
} else {
result.state = verifyStateCorruptHealed
result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, healed", k, err)
}
if healCancel != nil {
healCancel()
}
if !sendResult(result) {
return
}
continue
}
select {
case results <- "":
case <-ctx.Done():
// Block is valid
if !sendResult(&verifyResult{cid: k, state: verifyStateValid}) {
return
}
}
}
func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore) <-chan string {
results := make(chan string)
// verifyResultChan creates a channel of verification results by spawning multiple worker goroutines
// to process blocks in parallel. It returns immediately with a channel that will receive results.
func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) <-chan *verifyResult {
results := make(chan *verifyResult)
go func() {
defer close(results)
var wg sync.WaitGroup
for i := 0; i < runtime.NumCPU()*2; i++ {
for i := 0; i < runtime.NumCPU()*verifyWorkerMultiplier; i++ {
wg.Add(1)
go verifyWorkerRun(ctx, &wg, keys, results, bs)
go verifyWorkerRun(ctx, &wg, keys, results, bs, api, shouldDrop, shouldHeal, healTimeout)
}
wg.Wait()
@ -276,6 +370,45 @@ func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blocks
var repoVerifyCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Verify all blocks in repo are not corrupted.",
ShortDescription: `
'ipfs repo verify' checks integrity of all blocks in the local datastore.
Each block is read and validated against its CID to ensure data integrity.
Without any flags, this is a SAFE, read-only check that only reports corrupt
blocks without modifying the repository. This can be used as a "dry run" to
preview what --drop or --heal would do.
Use --drop to remove corrupt blocks, or --heal to remove and re-fetch from
the network.
Examples:
ipfs repo verify # safe read-only check, reports corrupt blocks
ipfs repo verify --drop # remove corrupt blocks
ipfs repo verify --heal # remove and re-fetch corrupt blocks
Exit Codes:
0: All blocks are valid, OR all corrupt blocks were successfully remediated
(with --drop or --heal)
1: Corrupt blocks detected (without flags), OR remediation failed (block
removal or healing failed with --drop or --heal)
Note: --heal requires the daemon to be running in online mode with network
connectivity to nodes that have the missing blocks. Make sure the daemon is
online and connected to other peers. Healing will attempt to re-fetch each
corrupt block from the network after removing it. If a block cannot be found
on the network, it will remain deleted.
WARNING: Both --drop and --heal are DESTRUCTIVE operations that permanently
delete corrupt blocks from your repository. Once deleted, blocks cannot be
recovered unless --heal successfully fetches them from the network. Blocks
that cannot be healed will remain permanently deleted. Always backup your
repository before using these options.
`,
},
Options: []cmds.Option{
cmds.BoolOption("drop", "Remove corrupt blocks from datastore (destructive operation)."),
cmds.BoolOption("heal", "Remove corrupt blocks and re-fetch from network (destructive operation, implies --drop)."),
cmds.StringOption("heal-timeout", "Maximum time to wait for each block heal (e.g., \"30s\"). Only applies with --heal.").WithDefault("30s"),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
nd, err := cmdenv.GetNode(env)
@ -283,6 +416,38 @@ var repoVerifyCmd = &cmds.Command{
return err
}
drop, _ := req.Options["drop"].(bool)
heal, _ := req.Options["heal"].(bool)
if heal {
drop = true // heal implies drop
}
// Parse and validate heal-timeout
timeoutStr, _ := req.Options["heal-timeout"].(string)
healTimeout, err := time.ParseDuration(timeoutStr)
if err != nil {
return fmt.Errorf("invalid heal-timeout: %w", err)
}
if healTimeout < 0 {
return errors.New("heal-timeout must be >= 0")
}
// Check online mode and API availability for healing operation
var api coreiface.CoreAPI
if heal {
if !nd.IsOnline {
return ErrNotOnline
}
api, err = cmdenv.GetApi(env, req)
if err != nil {
return err
}
if api == nil {
return fmt.Errorf("healing requested but API is not available - make sure daemon is online and connected to other peers")
}
}
bs := &bstore.ValidatingBlockstore{Blockstore: bstore.NewBlockstore(nd.Repo.Datastore())}
keys, err := bs.AllKeysChan(req.Context)
@ -291,17 +456,47 @@ var repoVerifyCmd = &cmds.Command{
return err
}
results := verifyResultChan(req.Context, keys, bs)
results := verifyResultChan(req.Context, keys, bs, api, drop, heal, healTimeout)
var fails int
// Track statistics for each type of outcome
var corrupted, removed, removeFailed, healed, healFailed int
var i int
for msg := range results {
if msg != "" {
if err := res.Emit(&VerifyProgress{Msg: msg}); err != nil {
for result := range results {
// Update counters based on the block's final state
switch result.state {
case verifyStateCorrupt:
// Block is corrupt but no action was taken (--drop not specified)
corrupted++
case verifyStateCorruptRemoved:
// Block was corrupt and successfully removed (--drop specified)
corrupted++
removed++
case verifyStateCorruptRemoveFailed:
// Block was corrupt but couldn't be removed
corrupted++
removeFailed++
case verifyStateCorruptHealed:
// Block was corrupt, removed, and successfully re-fetched (--heal specified)
corrupted++
removed++
healed++
case verifyStateCorruptHealFailed:
// Block was corrupt and removed, but re-fetching failed
corrupted++
removed++
healFailed++
default:
// verifyStateValid blocks are not counted (they're the expected case)
}
// Emit progress message for corrupt blocks
if result.state != verifyStateValid && result.msg != "" {
if err := res.Emit(&VerifyProgress{Msg: result.msg}); err != nil {
return err
}
fails++
}
i++
if err := res.Emit(&VerifyProgress{Progress: i}); err != nil {
return err
@ -312,8 +507,42 @@ var repoVerifyCmd = &cmds.Command{
return err
}
if fails != 0 {
return errors.New("verify complete, some blocks were corrupt")
if corrupted > 0 {
// Build a summary of what happened with corrupt blocks
summary := fmt.Sprintf("verify complete, %d blocks corrupt", corrupted)
if removed > 0 {
summary += fmt.Sprintf(", %d removed", removed)
}
if removeFailed > 0 {
summary += fmt.Sprintf(", %d failed to remove", removeFailed)
}
if healed > 0 {
summary += fmt.Sprintf(", %d healed", healed)
}
if healFailed > 0 {
summary += fmt.Sprintf(", %d failed to heal", healFailed)
}
// Determine success/failure based on operation mode
shouldFail := false
if !drop {
// Detection-only mode: always fail if corruption found
shouldFail = true
} else if heal {
// Heal mode: fail if any removal or heal failed
shouldFail = (removeFailed > 0 || healFailed > 0)
} else {
// Drop mode: fail if any removal failed
shouldFail = (removeFailed > 0)
}
if shouldFail {
return errors.New(summary)
}
// Success: emit summary as a message instead of error
return res.Emit(&VerifyProgress{Msg: summary})
}
return res.Emit(&VerifyProgress{Msg: "verify complete, all blocks validated."})
@ -322,7 +551,7 @@ var repoVerifyCmd = &cmds.Command{
Encoders: cmds.EncoderMap{
cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *VerifyProgress) error {
if strings.Contains(obj.Msg, "was corrupt") {
fmt.Fprintln(os.Stdout, obj.Msg)
fmt.Fprintln(w, obj.Msg)
return nil
}

View File

@ -0,0 +1,371 @@
//go:build go1.25
package commands
// This file contains unit tests for the --heal-timeout flag functionality
// using testing/synctest to avoid waiting for real timeouts.
//
// End-to-end tests for the full 'ipfs repo verify' command (including --drop
// and --heal flags) are located in test/cli/repo_verify_test.go.
import (
"bytes"
"context"
"errors"
"io"
"sync"
"testing"
"testing/synctest"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
coreiface "github.com/ipfs/kubo/core/coreiface"
"github.com/ipfs/kubo/core/coreiface/options"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ipfs/boxo/path"
)
func TestVerifyWorkerHealTimeout(t *testing.T) {
t.Run("heal succeeds before timeout", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const healTimeout = 5 * time.Second
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
// Setup channels
keys := make(chan cid.Cid, 1)
keys <- testCID
close(keys)
results := make(chan *verifyResult, 1)
// Mock blockstore that returns error (simulating corruption)
mockBS := &mockBlockstore{
getError: errors.New("corrupt block"),
}
// Mock API where Block().Get() completes before timeout
mockAPI := &mockCoreAPI{
blockAPI: &mockBlockAPI{
getDelay: 2 * time.Second, // Less than healTimeout
data: []byte("healed data"),
},
}
var wg sync.WaitGroup
wg.Add(1)
// Run worker
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
// Advance time past the mock delay but before timeout
time.Sleep(3 * time.Second)
synctest.Wait()
wg.Wait()
close(results)
// Verify heal succeeded
result := <-results
require.NotNil(t, result)
assert.Equal(t, verifyStateCorruptHealed, result.state)
assert.Contains(t, result.msg, "healed")
})
})
t.Run("heal fails due to timeout", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const healTimeout = 2 * time.Second
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
// Setup channels
keys := make(chan cid.Cid, 1)
keys <- testCID
close(keys)
results := make(chan *verifyResult, 1)
// Mock blockstore that returns error (simulating corruption)
mockBS := &mockBlockstore{
getError: errors.New("corrupt block"),
}
// Mock API where Block().Get() takes longer than healTimeout
mockAPI := &mockCoreAPI{
blockAPI: &mockBlockAPI{
getDelay: 5 * time.Second, // More than healTimeout
data: []byte("healed data"),
},
}
var wg sync.WaitGroup
wg.Add(1)
// Run worker
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
// Advance time past timeout
time.Sleep(3 * time.Second)
synctest.Wait()
wg.Wait()
close(results)
// Verify heal failed due to timeout
result := <-results
require.NotNil(t, result)
assert.Equal(t, verifyStateCorruptHealFailed, result.state)
assert.Contains(t, result.msg, "failed to heal")
assert.Contains(t, result.msg, "context deadline exceeded")
})
})
t.Run("heal with zero timeout still attempts heal", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const healTimeout = 0 // Zero timeout means no timeout
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
// Setup channels
keys := make(chan cid.Cid, 1)
keys <- testCID
close(keys)
results := make(chan *verifyResult, 1)
// Mock blockstore that returns error (simulating corruption)
mockBS := &mockBlockstore{
getError: errors.New("corrupt block"),
}
// Mock API that succeeds quickly
mockAPI := &mockCoreAPI{
blockAPI: &mockBlockAPI{
getDelay: 100 * time.Millisecond,
data: []byte("healed data"),
},
}
var wg sync.WaitGroup
wg.Add(1)
// Run worker
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
// Advance time to let heal complete
time.Sleep(200 * time.Millisecond)
synctest.Wait()
wg.Wait()
close(results)
// Verify heal succeeded even with zero timeout
result := <-results
require.NotNil(t, result)
assert.Equal(t, verifyStateCorruptHealed, result.state)
assert.Contains(t, result.msg, "healed")
})
})
t.Run("multiple blocks with different timeout outcomes", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const healTimeout = 3 * time.Second
testCID1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
testCID2 := cid.MustParse("bafybeihvvulpp4evxj7x7armbqcyg6uezzuig6jp3lktpbovlqfkjtgyby")
// Setup channels
keys := make(chan cid.Cid, 2)
keys <- testCID1
keys <- testCID2
close(keys)
results := make(chan *verifyResult, 2)
// Mock blockstore that always returns error (all blocks corrupt)
mockBS := &mockBlockstore{
getError: errors.New("corrupt block"),
}
// Create two mock block APIs with different delays
// We'll need to alternate which one gets used
// For simplicity, use one that succeeds fast
mockAPI := &mockCoreAPI{
blockAPI: &mockBlockAPI{
getDelay: 1 * time.Second, // Less than healTimeout - will succeed
data: []byte("healed data"),
},
}
var wg sync.WaitGroup
wg.Add(2) // Two workers
// Run two workers
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
// Advance time to let both complete
time.Sleep(2 * time.Second)
synctest.Wait()
wg.Wait()
close(results)
// Collect results
var healedCount int
for result := range results {
if result.state == verifyStateCorruptHealed {
healedCount++
}
}
// Both should heal successfully (both under timeout)
assert.Equal(t, 2, healedCount)
})
})
t.Run("valid block is not healed", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const healTimeout = 5 * time.Second
testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
// Setup channels
keys := make(chan cid.Cid, 1)
keys <- testCID
close(keys)
results := make(chan *verifyResult, 1)
// Mock blockstore that returns valid block (no error)
mockBS := &mockBlockstore{
block: blocks.NewBlock([]byte("valid data")),
}
// Mock API (won't be called since block is valid)
mockAPI := &mockCoreAPI{
blockAPI: &mockBlockAPI{},
}
var wg sync.WaitGroup
wg.Add(1)
// Run worker with heal enabled
go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, false, true, healTimeout)
synctest.Wait()
wg.Wait()
close(results)
// Verify block is marked valid, not healed
result := <-results
require.NotNil(t, result)
assert.Equal(t, verifyStateValid, result.state)
assert.Empty(t, result.msg)
})
})
}
// mockBlockstore implements a minimal blockstore for testing
type mockBlockstore struct {
getError error
block blocks.Block
}
func (m *mockBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
if m.getError != nil {
return nil, m.getError
}
return m.block, nil
}
func (m *mockBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
return nil
}
func (m *mockBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
return m.block != nil, nil
}
func (m *mockBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
if m.block != nil {
return len(m.block.RawData()), nil
}
return 0, errors.New("block not found")
}
func (m *mockBlockstore) Put(ctx context.Context, b blocks.Block) error {
return nil
}
func (m *mockBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
return nil
}
func (m *mockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, errors.New("not implemented")
}
func (m *mockBlockstore) HashOnRead(enabled bool) {
}
// mockBlockAPI implements BlockAPI for testing
type mockBlockAPI struct {
getDelay time.Duration
getError error
data []byte
}
func (m *mockBlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) {
if m.getDelay > 0 {
select {
case <-time.After(m.getDelay):
// Delay completed
case <-ctx.Done():
return nil, ctx.Err()
}
}
if m.getError != nil {
return nil, m.getError
}
return bytes.NewReader(m.data), nil
}
func (m *mockBlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (coreiface.BlockStat, error) {
return nil, errors.New("not implemented")
}
func (m *mockBlockAPI) Rm(ctx context.Context, p path.Path, opts ...options.BlockRmOption) error {
return errors.New("not implemented")
}
func (m *mockBlockAPI) Stat(ctx context.Context, p path.Path) (coreiface.BlockStat, error) {
return nil, errors.New("not implemented")
}
// mockCoreAPI implements minimal CoreAPI for testing
type mockCoreAPI struct {
blockAPI *mockBlockAPI
}
func (m *mockCoreAPI) Block() coreiface.BlockAPI {
return m.blockAPI
}
func (m *mockCoreAPI) Unixfs() coreiface.UnixfsAPI { return nil }
func (m *mockCoreAPI) Dag() coreiface.APIDagService { return nil }
func (m *mockCoreAPI) Name() coreiface.NameAPI { return nil }
func (m *mockCoreAPI) Key() coreiface.KeyAPI { return nil }
func (m *mockCoreAPI) Pin() coreiface.PinAPI { return nil }
func (m *mockCoreAPI) Object() coreiface.ObjectAPI { return nil }
func (m *mockCoreAPI) Swarm() coreiface.SwarmAPI { return nil }
func (m *mockCoreAPI) PubSub() coreiface.PubSubAPI { return nil }
func (m *mockCoreAPI) Routing() coreiface.RoutingAPI { return nil }
func (m *mockCoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) {
return path.ImmutablePath{}, nil, errors.New("not implemented")
}
func (m *mockCoreAPI) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) {
return nil, errors.New("not implemented")
}
func (m *mockCoreAPI) WithOptions(...options.ApiOption) (coreiface.CoreAPI, error) {
return nil, errors.New("not implemented")
}

View File

@ -211,6 +211,10 @@ var provideRefRoutingCmd = &cmds.Command{
ctx, events := routing.RegisterForQueryEvents(ctx)
var provideErr error
// TODO: not sure if necessary to call StartProviding for `ipfs routing
// provide <cid>`, since either cid is already being provided, or it will
// be garbage collected and not reprovided anyway. So we may simply stick
// with a single (optimistic) provide, and skip StartProviding call.
go func() {
defer cancel()
if rec {
@ -226,6 +230,16 @@ var provideRefRoutingCmd = &cmds.Command{
}
}()
if nd.HasActiveDHTClient() {
// If node has a DHT client, provide immediately the supplied cids before
// returning.
for _, c := range cids {
if err = provideCIDSync(req.Context, nd.DHTClient, c); err != nil {
return fmt.Errorf("error providing cid: %w", err)
}
}
}
for e := range events {
if err := res.Emit(e); err != nil {
return err
@ -300,6 +314,7 @@ func provideCids(prov node.DHTProvider, cids []cid.Cid) error {
for i, c := range cids {
mhs[i] = c.Hash()
}
// providing happens asynchronously
return prov.StartProviding(true, mhs...)
}

View File

@ -75,7 +75,8 @@ This interface is not stable and may change from release to release.
var dht *dht.IpfsDHT
var separateClient bool
if nd.DHTClient != nd.DHT {
// Check if using separate DHT client (e.g., accelerated DHT)
if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT {
separateClient = true
}

View File

@ -255,7 +255,7 @@ func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutp
}
// Amino DHT client keeps information about previously seen peers
if nd.DHTClient != nd.DHT && nd.DHTClient != nil {
if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT {
client, ok := nd.DHTClient.(*fullrt.FullRT)
if !ok {
return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")

View File

@ -30,9 +30,11 @@ import (
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
pubsub "github.com/libp2p/go-libp2p-pubsub"
psrouter "github.com/libp2p/go-libp2p-pubsub-router"
record "github.com/libp2p/go-libp2p-record"
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
connmgr "github.com/libp2p/go-libp2p/core/connmgr"
ic "github.com/libp2p/go-libp2p/core/crypto"
p2phost "github.com/libp2p/go-libp2p/core/host"
@ -143,6 +145,42 @@ func (n *IpfsNode) Close() error {
return n.stop()
}
// HasActiveDHTClient checks if the node's DHT client is active and usable for DHT operations.
//
// Returns false for:
// - nil DHTClient
// - typed nil pointers (e.g., (*ddht.DHT)(nil))
// - no-op routers (routinghelpers.Null)
//
// Note: This method only checks for known DHT client types (ddht.DHT, fullrt.FullRT).
// Custom routing.Routing implementations are not explicitly validated.
//
// This method prevents the "typed nil interface" bug where an interface contains
// a nil pointer of a concrete type, which passes nil checks but panics when methods
// are called.
func (n *IpfsNode) HasActiveDHTClient() bool {
if n.DHTClient == nil {
return false
}
// Check for no-op router (Routing.Type=none)
if _, ok := n.DHTClient.(routinghelpers.Null); ok {
return false
}
// Check for typed nil *ddht.DHT (common when Routing.Type=delegated or HTTP-only)
if d, ok := n.DHTClient.(*ddht.DHT); ok && d == nil {
return false
}
// Check for typed nil *fullrt.FullRT (accelerated DHT client)
if f, ok := n.DHTClient.(*fullrt.FullRT); ok && f == nil {
return false
}
return true
}
// Context returns the IpfsNode context
func (n *IpfsNode) Context() context.Context {
if n.ctx == nil {

View File

@ -1,15 +1,28 @@
package core
import (
"os"
"path/filepath"
"testing"
context "context"
"github.com/ipfs/kubo/repo"
"github.com/ipfs/boxo/filestore"
"github.com/ipfs/boxo/keystore"
datastore "github.com/ipfs/go-datastore"
syncds "github.com/ipfs/go-datastore/sync"
config "github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/node/libp2p"
golib "github.com/libp2p/go-libp2p"
ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
pstore "github.com/libp2p/go-libp2p/core/peerstore"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
)
func TestInitialization(t *testing.T) {
@ -65,3 +78,151 @@ var testIdentity = config.Identity{
PeerID: "QmNgdzLieYi8tgfo2WfTUzNVH5hQK9oAYGVf6dxN12NrHt",
PrivKey: "CAASrRIwggkpAgEAAoICAQCwt67GTUQ8nlJhks6CgbLKOx7F5tl1r9zF4m3TUrG3Pe8h64vi+ILDRFd7QJxaJ/n8ux9RUDoxLjzftL4uTdtv5UXl2vaufCc/C0bhCRvDhuWPhVsD75/DZPbwLsepxocwVWTyq7/ZHsCfuWdoh/KNczfy+Gn33gVQbHCnip/uhTVxT7ARTiv8Qa3d7qmmxsR+1zdL/IRO0mic/iojcb3Oc/PRnYBTiAZFbZdUEit/99tnfSjMDg02wRayZaT5ikxa6gBTMZ16Yvienq7RwSELzMQq2jFA4i/TdiGhS9uKywltiN2LrNDBcQJSN02pK12DKoiIy+wuOCRgs2NTQEhU2sXCk091v7giTTOpFX2ij9ghmiRfoSiBFPJA5RGwiH6ansCHtWKY1K8BS5UORM0o3dYk87mTnKbCsdz4bYnGtOWafujYwzueGx8r+IWiys80IPQKDeehnLW6RgoyjszKgL/2XTyP54xMLSW+Qb3BPgDcPaPO0hmop1hW9upStxKsefW2A2d46Ds4HEpJEry7PkS5M4gKL/zCKHuxuXVk14+fZQ1rstMuvKjrekpAC2aVIKMI9VRA3awtnje8HImQMdj+r+bPmv0N8rTTr3eS4J8Yl7k12i95LLfK+fWnmUh22oTNzkRlaiERQrUDyE4XNCtJc0xs1oe1yXGqazCIAQIDAQABAoICAQCk1N/ftahlRmOfAXk//8wNl7FvdJD3le6+YSKBj0uWmN1ZbUSQk64chr12iGCOM2WY180xYjy1LOS44PTXaeW5bEiTSnb3b3SH+HPHaWCNM2EiSogHltYVQjKW+3tfH39vlOdQ9uQ+l9Gh6iTLOqsCRyszpYPqIBwi1NMLY2Ej8PpVU7ftnFWouHZ9YKS7nAEiMoowhTu/7cCIVwZlAy3AySTuKxPMVj9LORqC32PVvBHZaMPJ+X1Xyijqg6aq39WyoztkXg3+Xxx5j5eOrK6vO/Lp6ZUxaQilHDXoJkKEJjgIBDZpluss08UPfOgiWAGkW+L4fgUxY0qDLDAEMhyEBAn6KOKVL1JhGTX6GjhWziI94bddSpHKYOEIDzUy4H8BXnKhtnyQV6ELS65C2hj9D0IMBTj7edCF1poJy0QfdK0cuXgMvxHLeUO5uc2YWfbNosvKxqygB9rToy4b22YvNwsZUXsTY6Jt+p9V2OgXSKfB5VPeRbjTJL6xqvvUJpQytmII/C9JmSDUtCbYceHj6X9jgigLk20VV6nWHqCTj3utXD6NPAjoycVpLKDlnWEgfVELDIk0gobxUqqSm3jTPEKRPJgxkgPxbwxYumtw++1UY2y35w3WRDc2xYPaWKBCQeZy+mL6ByXp9bWlNvxS3Knb6oZp36/ovGnf2pGvdQKCAQEAyKpipz2lIUySDyE0avVWAmQb2tWGKXALPohzj7AwkcfEg2GuwoC6GyVE2sTJD1HRazIjOKn3yQORg2uOPeG7sx7EKHxSxCKDrbPawkvLCq8JYSy9TLvhqKUVVGYPqMBzu2POSLEA81QXas+aYjKOFWA2Zrjq26zV9ey3+6Lc6WULePgRQybU8+RHJc6fdjUCCfUxgOrUO2IQOuTJ+FsDpVnrMUGlokmWn23OjL4qTL9wGDnWGUs2pjSzNbj3qA0d8iqaiMUyHX/D/VS0wpeT1osNBSm8suvSibYBn+7wbIApbwXUxZaxMv2OHGz3empae4ckvNZs7r8wsI9UwFt8mwKCAQEA4XK6gZkv9t+3YCcSPw2ensLvL/xU7i2bkC9tfTGdjnQfzZXIf5KNdVuj/SerOl2S1s45NMs3ysJbADwRb4ahElD/V71nGzV8fpFTitC20ro9fuX4J0+twmBolHqeH9pmeGTjAeL1rvt6vxs4FkeG/yNft7GdXpXTtEGaObn8Mt0tPY+aB3UnKrnCQoQAlPyGHFrVRX0UEcp6wyyNGhJCNKeNOvqCHTFObhbhO+KWpWSN0MkVHnqaIBnIn1Te8FtvP/iTwXGnKc0YXJUG6+LM6LmOguW6tg8ZqiQeYyyR+e9eCFH4csLzkrTl1GxCxwEsoSLIMm7UDcjttW6tYEghkwKCAQEAmeCO5lCPYImnN5Lu71ZTLmI2OgmjaANTnBBnDbi+hgv61gUCToUIMejSdDCTPfwv61P3TmyIZs0luPGxkiKYHTNqmOE9Vspgz8Mr7fLRMNApESuNvloVIY32XVImj/GEzh4rAfM6F15U1sN8T/EUo6+0B/Glp+9R49QzAfRSE2g48/rGwgf1JVHYfVWFUtAzUA+GdqWdOixo5cCsYJbqpNHfWVZN/bUQnBFIYwUwysnC29D+LUdQEQQ4qOm+gFAOtrWU62zMkXJ4iLt8Ify6kbrvsRXgbhQIzzGS7WH9XDarj0eZciuslr15TLMC1Azadf+cXHLR9gMHA13mT9vYIQKCAQA/DjGv8cKCkAvf7s2hqROGYAs6Jp8yhrsN1tYOwAPLRhtnCs+rLrg17M2vDptLlcRuI/vIElamdTmylRpjUQpX7yObzLO73nfVhpwRJVMdGU394iBIDncQ+JoHfUwgqJskbUM40dvZdyjbrqc/Q/4z+hbZb+oN/GXb8sVKBATPzSDMKQ/xqgisYIw+wmDPStnPsHAaIWOtni47zIgilJzD0WEk78/YjmPbUrboYvWziK5JiRRJFA1rkQqV1c0M+OXixIm+/yS8AksgCeaHr0WUieGcJtjT9uE8vyFop5ykhRiNxy9wGaq6i7IEecsrkd6DqxDHWkwhFuO1bSE83q/VAoIBAEA+RX1i/SUi08p71ggUi9WFMqXmzELp1L3hiEjOc2AklHk2rPxsaTh9+G95BvjhP7fRa/Yga+yDtYuyjO99nedStdNNSg03aPXILl9gs3r2dPiQKUEXZJ3FrH6tkils/8BlpOIRfbkszrdZIKTO9GCdLWQ30dQITDACs8zV/1GFGrHFrqnnMe/NpIFHWNZJ0/WZMi8wgWO6Ik8jHEpQtVXRiXLqy7U6hk170pa4GHOzvftfPElOZZjy9qn7KjdAQqy6spIrAE94OEL+fBgbHQZGLpuTlj6w6YGbMtPU8uo7sXKoc6WOCb68JWft3tejGLDa1946HAWqVM9B/UcneNc=",
}
// mockHostOption creates a HostOption that uses the provided mocknet.
// Inlined to avoid import cycle with core/mock package.
func mockHostOption(mn mocknet.Mocknet) libp2p.HostOption {
return func(id peer.ID, ps pstore.Peerstore, opts ...golib.Option) (host.Host, error) {
var cfg golib.Config
if err := cfg.Apply(opts...); err != nil {
return nil, err
}
// The mocknet does not use the provided libp2p.Option. This options include
// the listening addresses we want our peer listening on. Therefore, we have
// to manually parse the configuration and add them here.
ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL)
return mn.AddPeerWithPeerstore(id, ps)
}
}
func TestHasActiveDHTClient(t *testing.T) {
// Test 1: nil DHTClient
t.Run("nil DHTClient", func(t *testing.T) {
node := &IpfsNode{
DHTClient: nil,
}
if node.HasActiveDHTClient() {
t.Error("Expected false for nil DHTClient")
}
})
// Test 2: Typed nil *ddht.DHT (common case when Routing.Type=delegated)
t.Run("typed nil ddht.DHT", func(t *testing.T) {
node := &IpfsNode{
DHTClient: (*ddht.DHT)(nil),
}
if node.HasActiveDHTClient() {
t.Error("Expected false for typed nil *ddht.DHT")
}
})
// Test 3: Typed nil *fullrt.FullRT (accelerated DHT client)
t.Run("typed nil fullrt.FullRT", func(t *testing.T) {
node := &IpfsNode{
DHTClient: (*fullrt.FullRT)(nil),
}
if node.HasActiveDHTClient() {
t.Error("Expected false for typed nil *fullrt.FullRT")
}
})
// Test 4: routinghelpers.Null no-op router (Routing.Type=none)
t.Run("routinghelpers.Null", func(t *testing.T) {
node := &IpfsNode{
DHTClient: routinghelpers.Null{},
}
if node.HasActiveDHTClient() {
t.Error("Expected false for routinghelpers.Null")
}
})
// Test 5: Valid standard dual DHT (Routing.Type=auto/dht/dhtclient)
t.Run("valid standard dual DHT", func(t *testing.T) {
ctx := context.Background()
mn := mocknet.New()
defer mn.Close()
ds := syncds.MutexWrap(datastore.NewMapDatastore())
c := config.Config{}
c.Identity = testIdentity
c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"}
r := &repo.Mock{
C: c,
D: ds,
K: keystore.NewMemKeystore(),
F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())),
}
node, err := NewNode(ctx, &BuildCfg{
Routing: libp2p.DHTServerOption,
Repo: r,
Host: mockHostOption(mn),
Online: true,
})
if err != nil {
t.Fatalf("Failed to create node with DHT: %v", err)
}
defer node.Close()
// First verify test setup created the expected DHT type
if node.DHTClient == nil {
t.Fatalf("Test setup failed: DHTClient is nil")
}
if _, ok := node.DHTClient.(*ddht.DHT); !ok {
t.Fatalf("Test setup failed: expected DHTClient to be *ddht.DHT, got %T", node.DHTClient)
}
// Now verify HasActiveDHTClient() correctly identifies it as active
if !node.HasActiveDHTClient() {
t.Error("Expected true for valid dual DHT client")
}
})
// Test 6: Valid accelerated DHT client (Routing.Type=autoclient)
t.Run("valid accelerated DHT client", func(t *testing.T) {
ctx := context.Background()
mn := mocknet.New()
defer mn.Close()
ds := syncds.MutexWrap(datastore.NewMapDatastore())
c := config.Config{}
c.Identity = testIdentity
c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"}
c.Routing.AcceleratedDHTClient = config.True
r := &repo.Mock{
C: c,
D: ds,
K: keystore.NewMemKeystore(),
F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())),
}
node, err := NewNode(ctx, &BuildCfg{
Routing: libp2p.DHTOption,
Repo: r,
Host: mockHostOption(mn),
Online: true,
})
if err != nil {
t.Fatalf("Failed to create node with accelerated DHT: %v", err)
}
defer node.Close()
// First verify test setup created the expected accelerated DHT type
if node.DHTClient == nil {
t.Fatalf("Test setup failed: DHTClient is nil")
}
if _, ok := node.DHTClient.(*fullrt.FullRT); !ok {
t.Fatalf("Test setup failed: expected DHTClient to be *fullrt.FullRT, got %T", node.DHTClient)
}
// Now verify HasActiveDHTClient() correctly identifies it as active
if !node.HasActiveDHTClient() {
t.Error("Expected true for valid accelerated DHT client")
}
})
}

View File

@ -240,14 +240,27 @@ func (tp *TestSuite) TestRoutingProvide(t *testing.T) {
t.Fatal(err)
}
out, err = apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1))
if err != nil {
t.Fatal(err)
maxAttempts := 5
success := false
for range maxAttempts {
// We may need to try again as Provide() doesn't block until the CID is
// actually provided.
out, err = apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1))
if err != nil {
t.Fatal(err)
}
provider := <-out
if provider.ID.String() == self0.ID().String() {
success = true
break
}
if len(provider.ID.String()) > 0 {
t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String())
}
time.Sleep(time.Second)
}
provider := <-out
if provider.ID.String() != self0.ID().String() {
t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String())
if !success {
t.Errorf("missing provider after %d attempts", maxAttempts)
}
}

View File

@ -55,12 +55,24 @@ func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (out P2PHo
return out, err
}
// Optimistic provide is enabled either via dedicated expierimental flag, or when DHT Provide Sweep is enabled.
// When DHT Provide Sweep is enabled, all provide operations go through the
// `SweepingProvider`, hence the provides don't use the optimistic provide
// logic. Provides use `SweepingProvider.StartProviding()` and not
// `IpfsDHT.Provide()`, which is where the optimistic provide logic is
// implemented. However, `IpfsDHT.Provide()` is used to quickly provide roots
// when user manually adds content with the `--fast-provide` flag enabled. In
// this case we want to use optimistic provide logic to quickly announce the
// content to the network. This should be the only use case of
// `IpfsDHT.Provide()` when DHT Provide Sweep is enabled.
optimisticProvide := cfg.Experimental.OptimisticProvide || cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled)
routingOptArgs := RoutingOptionArgs{
Ctx: ctx,
Datastore: params.Repo.Datastore(),
Validator: params.Validator,
BootstrapPeers: bootstrappers,
OptimisticProvide: cfg.Experimental.OptimisticProvide,
OptimisticProvide: optimisticProvide,
OptimisticProvideJobsPoolSize: cfg.Experimental.OptimisticProvideJobsPoolSize,
LoopbackAddressesOnLanDHT: cfg.Routing.LoopbackAddressesOnLanDHT.WithDefault(config.DefaultLoopbackAddressesOnLanDHT),
}

View File

@ -116,6 +116,7 @@ type DHTProvider interface {
// `OfflineDelay`). The schedule depends on the network size, hence recent
// network connectivity is essential.
RefreshSchedule() error
Close() error
}
var (
@ -134,6 +135,7 @@ func (r *NoopProvider) StartProviding(bool, ...mh.Multihash) error { return nil
func (r *NoopProvider) ProvideOnce(...mh.Multihash) error { return nil }
func (r *NoopProvider) Clear() int { return 0 }
func (r *NoopProvider) RefreshSchedule() error { return nil }
func (r *NoopProvider) Close() error { return nil }
// LegacyProvider is a wrapper around the boxo/provider.System that implements
// the DHTProvider interface. This provider manages reprovides using a burst
@ -523,8 +525,41 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
case <-ctx.Done():
return ctx.Err()
}
// Keystore data isn't purged, on close, but it will be overwritten
// when the node starts again.
// Keystore will be closed by ensureProviderClosesBeforeKeystore hook
// to guarantee provider closes before keystore.
return nil
},
})
})
// ensureProviderClosesBeforeKeystore manages the shutdown order between
// provider and keystore to prevent race conditions.
//
// The provider's worker goroutines may call keystore methods during their
// operation. If keystore closes while these operations are in-flight, we get
// "keystore is closed" errors. By closing the provider first, we ensure all
// worker goroutines exit and complete any pending keystore operations before
// the keystore itself closes.
type providerKeystoreShutdownInput struct {
fx.In
Provider DHTProvider
Keystore *keystore.ResettableKeystore
}
ensureProviderClosesBeforeKeystore := fx.Invoke(func(lc fx.Lifecycle, in providerKeystoreShutdownInput) {
// Skip for NoopProvider
if _, ok := in.Provider.(*NoopProvider); ok {
return
}
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
// Close provider first - waits for all worker goroutines to exit.
// This ensures no code can access keystore after this returns.
if err := in.Provider.Close(); err != nil {
logger.Errorw("error closing provider during shutdown", "error", err)
}
// Close keystore - safe now, provider is fully shut down
return in.Keystore.Close()
},
})
@ -650,6 +685,7 @@ See docs: https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtmaxw
return fx.Options(
sweepingReprovider,
initKeystore,
ensureProviderClosesBeforeKeystore,
reprovideAlert,
)
}

View File

@ -59,6 +59,9 @@ A new experimental DHT provider is available as an alternative to both the defau
**Monitoring and debugging:** Legacy mode (`SweepEnabled=false`) tracks `provider_reprovider_provide_count` and `provider_reprovider_reprovide_count`, while sweep mode (`SweepEnabled=true`) tracks `total_provide_count_total`. Enable debug logging with `GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug` to see detailed logs from either system.
> [!IMPORTANT]
> The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly.
> [!NOTE]
> This feature is experimental and opt-in. In the future, it will become the default and replace the legacy system. Some commands like `ipfs stats provide` and `ipfs routing provide` are not yet available with sweep mode. Run `ipfs provide --help` for alternatives.
@ -68,6 +71,9 @@ For configuration details, see [`Provide.DHT`](https://github.com/ipfs/kubo/blob
Kubo now exposes DHT metrics from [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/), including `total_provide_count_total` for sweep provider operations and RPC metrics prefixed with `rpc_inbound_` and `rpc_outbound_` for DHT message traffic. See [Kubo metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md) for details.
> [!IMPORTANT]
> The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly.
#### 🚨 Improved gateway error pages with diagnostic tools
Gateway error pages now provide more actionable information during content retrieval failures. When a 504 Gateway Timeout occurs, users see detailed retrieval state information including which phase failed and a sample of providers that were attempted:

View File

@ -10,11 +10,15 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [🎯 Amino DHT Sweep provider is now the default](#-amino-dht-sweep-provider-is-now-the-default)
- [⚡ Fast root CID providing for immediate content discovery](#-fast-root-cid-providing-for-immediate-content-discovery)
- [📊 Detailed statistics for Sweep provider with `ipfs provide stat`](#-detailed-statistics-for-sweep-provider-with-ipfs-provide-stat)
- [⏯️ Provider resume cycle for improved reproviding reliability](#provider-resume-cycle-for-improved-reproviding-reliability)
- [🔔 Sweep provider slow reprovide warnings](#-sweep-provider-slow-reprovide-warnings)
- [📊 Metric rename: `provider_provides_total`](#-metric-rename-provider_provides_total)
- [🔧 Fixed UPnP port forwarding after router restarts](#-fixed-upnp-port-forwarding-after-router-restarts)
- [🖥️ RISC-V support with prebuilt binaries](#-risc-v-support-with-prebuilt-binaries)
- [🚦 Gateway range request limits for CDN compatibility](#-gateway-range-request-limits-for-cdn-compatibility)
- [🪦 Deprecated `go-ipfs` name no longer published](#-deprecated-go-ipfs-name-no-longer-published)
- [📦️ Important dependency updates](#-important-dependency-updates)
- [📝 Changelog](#-changelog)
@ -22,77 +26,77 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
### Overview
Kubo 0.39.0 graduates the experimental sweep provider to default, bringing efficient content announcement to all nodes. This release adds fast root CID providing for immediate content discovery via `ipfs add`, detailed provider statistics, automatic state persistence for reliable reproviding after restarts, and proactive monitoring alerts for identifying issues early. It also includes important fixes for UPnP port forwarding, RISC-V prebuilt binaries, and finalizes the deprecation of the legacy go-ipfs name.
### 🔦 Highlights
#### 🚦 Gateway range request limits for CDN compatibility
#### 🎯 Amino DHT Sweep provider is now the default
The new [`Gateway.MaxRangeRequestFileSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) configuration protects against CDN bugs where range requests over a certain size are silently ignored and the entire file is returned instead ([boxo#856](https://github.com/ipfs/boxo/issues/856#issuecomment-2786431369)). This causes unexpected bandwidth costs for both gateway operators and clients who only wanted a small byte range.
The Amino DHT Sweep provider system, introduced as experimental in v0.38, is now enabled by default (`Provide.DHT.SweepEnabled=true`).
**What this means:** All nodes now benefit from efficient keyspace-sweeping content announcements that reduce memory overhead and create predictable network patterns, especially for nodes providing large content collections.
**Migration:** The transition is automatic on upgrade. Your existing configuration is preserved:
- If you explicitly set `Provide.DHT.SweepEnabled=false` in v0.38, you'll continue using the legacy provider
- If you were using the default settings, you'll automatically get the sweep provider
- To opt out and return to legacy behavior: `ipfs config --json Provide.DHT.SweepEnabled false`
**New features available with sweep mode:**
- Detailed statistics via `ipfs provide stat` ([see below](#-detailed-statistics-for-sweep-provider-with-ipfs-provide-stat))
- Automatic resume after restarts with persistent state ([see below](#provider-resume-cycle-for-improved-reproviding-reliability))
- Proactive alerts when reproviding falls behind ([see below](#-sweep-provider-slow-reprovide-warnings))
- Better metrics for monitoring (`provider_provides_total`) ([see below](#-metric-rename-provider_provides_total))
- Fast optimistic provide of new root CIDs ([see below](#-fast-root-cid-providing-for-immediate-content-discovery))
For background on the sweep provider design and motivations, see [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled) and [ipshipyard.com#8](https://github.com/ipshipyard/ipshipyard.com/pull/8).
#### ⚡ Fast root CID providing for immediate content discovery
When you add content to IPFS, the sweep provider queues it for efficient DHT provides over time. While this is resource-efficient, other peers won't find your content immediately after `ipfs add` or `ipfs dag import` completes.
To make sharing faster, `ipfs add` and `ipfs dag import` now do an immediate provide of root CIDs to the DHT in addition to the regular queue (controlled by the new `--fast-provide-root` flag, enabled by default). This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to `Provide.Strategy` over time.
This closes the gap between command completion and content shareability: root CIDs typically become discoverable on the network in under a second (compared to 30+ seconds previously). The feature uses optimistic DHT operations, which are significantly faster with the sweep provider (now enabled by default).
By default, this immediate provide runs in the background without blocking the command. For use cases requiring guaranteed discoverability before the command returns (e.g., sharing a link immediately), use `--fast-provide-wait` to block until the provide completes.
**Simple examples:**
```bash
ipfs add file.txt # Root provided immediately, blocks queued for sweep provider
ipfs add file.txt --fast-provide-wait # Wait for root provide to complete
ipfs dag import file.car # Same for CAR imports
```
**Configuration:** Set defaults via `Import.FastProvideRoot` (default: `true`) and `Import.FastProvideWait` (default: `false`). See `ipfs add --help` and `ipfs dag import --help` for more details and examples.
This optimization works best with the sweep provider and accelerated DHT client, where provide operations are significantly faster. Automatically skipped when DHT is unavailable (e.g., `Routing.Type=none` or delegated-only configurations).
Set this to your CDN's range request limit (e.g., `"5GiB"` for Cloudflare's default plan) to return 501 Not Implemented for oversized range requests, with an error message suggesting verifiable block requests as an alternative.
#### 📊 Detailed statistics for Sweep provider with `ipfs provide stat`
The experimental Sweep provider system ([introduced in
v0.38](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.38.md#-experimental-sweeping-dht-provider))
now has detailed statistics available through `ipfs provide stat`.
The Sweep provider system now exposes detailed statistics through `ipfs provide stat`, helping you monitor provider health and troubleshoot issues.
These statistics help you monitor provider health and troubleshoot issues,
especially useful for nodes providing large content collections. You can quickly
identify bottlenecks like queue backlog, worker saturation, or connectivity
problems that might prevent content from being announced to the DHT.
Run `ipfs provide stat` for a quick summary, or use `--all` to see complete metrics including connectivity status, queue sizes, reprovide schedules, network statistics, operation rates, and worker utilization. For real-time monitoring, use `watch ipfs provide stat --all --compact` to observe changes in a 2-column layout. Individual sections can be displayed with flags like `--network`, `--operations`, or `--workers`.
**Default behavior:** Displays a brief summary showing queue sizes, scheduled
CIDs/regions, average record holders, ongoing/total provides, and worker status
when resources are constrained.
For Dual DHT configurations, use `--lan` to view LAN DHT statistics instead of the default WAN DHT stats.
**Detailed statistics with `--all`:** View complete metrics organized into sections:
- **Connectivity**: DHT connection status
- **Queues**: Pending provide and reprovide operations
- **Schedule**: CIDs/regions scheduled for reprovide
- **Timings**: Uptime, reprovide cycle information
- **Network**: Peer statistics, keyspace region sizes
- **Operations**: Ongoing and past provides, rates, errors
- **Workers**: Worker pool utilization and availability
**Real-time monitoring:** For continuous monitoring, run
`watch ipfs provide stat --all --compact` to see detailed statistics refreshed
in a 2-column layout. This lets you observe provide rates, queue sizes, and
worker availability in real-time. Individual sections can be displayed using
flags like `--network`, `--operations`, or `--workers`, and multiple flags can
be combined for custom views.
**Dual DHT support:** For Dual DHT configurations, use `--lan` to view LAN DHT
provider statistics instead of the default WAN DHT stats.
For more information, run `ipfs provide stat --help` or see the [Provide Stats documentation](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md).
> [!NOTE]
> These statistics are only available when using the Sweep provider system
> (enabled via
> [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled)).
> Legacy provider shows basic statistics without flag support.
> Legacy provider (when `Provide.DHT.SweepEnabled=false`) shows basic statistics without flag support.
#### ⏯️ Provider resume cycle for improved reproviding reliability
When using the sweeping provider (`Provide.DHT.SweepEnabled`), Kubo now
persists the reprovide cycle state and automatically resumes where it left off
after a restart. This brings several improvements:
The Sweep provider now persists the reprovide cycle state and automatically resumes where it left off after a restart. This brings several improvements:
- **Persistent progress**: The provider now saves its position in the reprovide
cycle to the datastore. On restart, it continues from where it stopped instead
of starting from scratch.
- **Catch-up reproviding**: If the node was offline for an extended period, all
CIDs that haven't been reprovided within the configured reprovide interval are
immediately queued for reproviding when the node starts up. This ensures
content availability is maintained even after downtime.
- **Persistent provide queue**: The provide queue is now persisted to the
datastore on shutdown. When the node restarts, queued CIDs are restored and
provided as expected, preventing loss of pending provide operations.
- **Resume control**: The resume behavior is now controlled via the
`Provide.DHT.ResumeEnabled` config option (default: `true`). If you don't want
to keep the persisted provider state from a previous run, you can set
`Provide.DHT.ResumeEnabled=false` in your config.
- **Persistent progress**: The provider saves its position in the reprovide cycle to the datastore. On restart, it continues from where it stopped instead of starting from scratch.
- **Catch-up reproviding**: If the node was offline for an extended period, all CIDs that haven't been reprovided within the configured reprovide interval are immediately queued for reproviding when the node starts up. This ensures content availability is maintained even after downtime.
- **Persistent provide queue**: The provide queue is persisted to the datastore on shutdown. When the node restarts, queued CIDs are restored and provided as expected, preventing loss of pending provide operations.
- **Resume control**: The resume behavior is controlled via [`Provide.DHT.ResumeEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtresumeenabled) (default: `true`). Set to `false` if you don't want to keep the persisted provider state from a previous run.
This feature significantly improves the reliability of content providing,
especially for nodes that experience intermittent connectivity or restarts.
This feature improves reliability for nodes that experience intermittent connectivity or restarts.
#### 🔔 Sweep provider slow reprovide warnings
@ -110,6 +114,12 @@ The alert polls every 15 minutes (to avoid alert fatigue while catching
persistent issues) and only triggers after sustained growth across multiple
intervals. The legacy provider is unaffected by this change.
#### 📊 Metric rename: `provider_provides_total`
The Amino DHT Sweep provider metric has been renamed from `total_provide_count_total` to `provider_provides_total` to follow OpenTelemetry naming conventions and maintain consistency with other kad-dht metrics (which use dot notation like `rpc.inbound.messages`, `rpc.outbound.requests`, etc.).
**Migration:** If you have Prometheus queries, dashboards, or alerts monitoring the old `total_provide_count_total` metric, update them to use `provider_provides_total` instead. This affects all nodes using sweep mode, which is now the default in v0.39 (previously opt-in experimental in v0.38).
#### 🔧 Fixed UPnP port forwarding after router restarts
Kubo now automatically recovers UPnP port mappings when routers restart or
@ -136,26 +146,27 @@ using UPnP for NAT traversal.
#### 🖥️ RISC-V support with prebuilt binaries
Kubo now provides official `linux-riscv64` prebuilt binaries with every release,
bringing IPFS to [RISC-V](https://en.wikipedia.org/wiki/RISC-V) open hardware.
Kubo provides official `linux-riscv64` prebuilt binaries, bringing IPFS to [RISC-V](https://en.wikipedia.org/wiki/RISC-V) open hardware.
As RISC-V single-board computers and embedded systems become more accessible,
it's good to see the distributed web supported on open hardware architectures -
a natural pairing of open technologies.
As RISC-V single-board computers and embedded systems become more accessible, the distributed web is now supported on open hardware architectures - a natural pairing of open technologies.
Download from <https://dist.ipfs.tech/kubo/> or
<https://github.com/ipfs/kubo/releases> and look for the `linux-riscv64` archive.
Download from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases> and look for the `linux-riscv64` archive.
#### 🚦 Gateway range request limits for CDN compatibility
The new [`Gateway.MaxRangeRequestFileSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) configuration protects against CDN range request limitations that cause bandwidth overcharges on deserialized responses. Some CDNs convert range requests over large files into full file downloads, causing clients requesting small byte ranges to unknowingly download entire multi-gigabyte files.
This only impacts deserialized responses. Clients using verifiable block requests (`application/vnd.ipld.raw`) are not affected. See the [configuration documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) for details.
#### 🪦 Deprecated `go-ipfs` name no longer published
The `go-ipfs` name was deprecated in 2022 and renamed to `kubo`. Starting with this release, we have stopped publishing Docker images and distribution binaries under the old `go-ipfs` name.
The `go-ipfs` name was deprecated in 2022 and renamed to `kubo`. Starting with this release, the legacy Docker image name has been replaced with a stub that displays an error message directing users to switch to `ipfs/kubo`.
Existing users should switch to:
**Docker images:** The `ipfs/go-ipfs` image tags now contain only a stub script that exits with an error, instructing users to update their Docker configurations to use [`ipfs/kubo`](https://hub.docker.com/r/ipfs/kubo) instead. This ensures users are aware of the deprecation while allowing existing automation to fail explicitly rather than silently using outdated images.
- Docker: `ipfs/kubo` image (instead of `ipfs/go-ipfs`)
- Binaries: download from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases>
**Distribution binaries:** Download Kubo from <https://dist.ipfs.tech/kubo/> or <https://github.com/ipfs/kubo/releases>. The legacy `go-ipfs` distribution path should no longer be used.
For Docker users, the legacy `ipfs/go-ipfs` image name now shows a deprecation notice directing you to `ipfs/kubo`.
All users should migrate to the `kubo` name in their scripts and configurations.
#### Routing V1 HTTP API now exposed by default

22
docs/changelogs/v0.40.md Normal file
View File

@ -0,0 +1,22 @@
# Kubo changelog v0.40
<a href="https://ipshipyard.com/"><img align="right" src="https://github.com/user-attachments/assets/39ed3504-bb71-47f6-9bf8-cb9a1698f272" /></a>
This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
- [v0.40.0](#v0400)
## v0.40.0
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
### Overview
### 🔦 Highlights
### 📝 Changelog
### 👨‍👩‍👧‍👦 Contributors

View File

@ -230,6 +230,8 @@ config file at runtime.
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
- [`Import.UnixFSChunker`](#importunixfschunker)
- [`Import.HashFunction`](#importhashfunction)
- [`Import.FastProvideRoot`](#importfastprovideroot)
- [`Import.FastProvideWait`](#importfastprovidewait)
- [`Import.BatchMaxNodes`](#importbatchmaxnodes)
- [`Import.BatchMaxSize`](#importbatchmaxsize)
- [`Import.UnixFSFileMaxLinks`](#importunixfsfilemaxlinks)
@ -1162,11 +1164,20 @@ Type: `optionalDuration`
### `Gateway.MaxRangeRequestFileSize`
Maximum file size for HTTP range requests. Range requests for files larger than this limit return 501 Not Implemented.
Maximum file size for HTTP range requests on deserialized responses. Range requests for files larger than this limit return 501 Not Implemented.
Protects against CDN bugs where range requests are silently ignored and the entire file is returned instead. For example, Cloudflare's default plan returns the full file for range requests over 5GiB, causing unexpected bandwidth costs for both gateway operators and clients who only wanted a small byte range.
**Why this exists:**
Set this to your CDN's range request limit (e.g., `"5GiB"` for Cloudflare's default plan). The error response suggests using verifiable block requests (application/vnd.ipld.raw) as an alternative.
Some CDNs like Cloudflare intercept HTTP range requests and convert them to full file downloads when files exceed their cache bucket limits. Cloudflare's default plan only caches range requests for files up to 5GiB. Files larger than this receive HTTP 200 with the entire file instead of HTTP 206 with the requested byte range. A client requesting 1MB from a 40GiB file would unknowingly download all 40GiB, causing bandwidth overcharges for the gateway operator, unexpected data costs for the client, and potential browser crashes.
This only affects deserialized responses. Clients fetching verifiable blocks as `application/vnd.ipld.raw` are not impacted because they work with small chunks that stay well below CDN cache limits.
**How to use:**
Set this to your CDN's range request cache limit (e.g., `"5GiB"` for Cloudflare's default plan). The gateway returns 501 Not Implemented for range requests over files larger than this limit, with an error message suggesting verifiable block requests as an alternative.
> [!NOTE]
> Cloudflare users running open gateway hosting deserialized responses should deploy additional protection via Cloudflare Snippets (requires Enterprise plan). The Kubo configuration alone is not sufficient because Cloudflare has already intercepted and cached the response by the time it reaches your origin. See [boxo#856](https://github.com/ipfs/boxo/issues/856#issuecomment-3523944976) for a snippet that aborts HTTP 200 responses when Content-Length exceeds the limit.
Default: `0` (no limit)
@ -2181,10 +2192,9 @@ to `false`.
You can compare the effectiveness of sweep mode vs legacy mode by monitoring the appropriate metrics (see [Monitoring Provide Operations](#monitoring-provide-operations) above).
> [!NOTE]
> This feature is opt-in for now, but will become the default in a future release.
> Eventually, this configuration flag will be removed once the feature is stable.
> This is the default provider system as of Kubo v0.39. To use the legacy provider instead, set `Provide.DHT.SweepEnabled=false`.
Default: `false`
Default: `true`
Type: `flag`
@ -3611,6 +3621,38 @@ Default: `sha2-256`
Type: `optionalString`
### `Import.FastProvideRoot`
Immediately provide root CIDs to the DHT in addition to the regular provide queue.
This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to the `Provide.Strategy` over time. Together, they optimize for both immediate discoverability of newly imported content and efficient resource usage for complete DAG provides.
When disabled, only the sweep provider's queue is used.
This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-root` flag.
Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
Default: `true`
Type: `flag`
### `Import.FastProvideWait`
Wait for the immediate root CID provide to complete before returning.
When enabled, the command blocks until the provide completes, ensuring guaranteed discoverability before returning. When disabled (default), the provide happens asynchronously in the background without blocking the command.
Use this when you need certainty that content is discoverable before the command returns (e.g., sharing a link immediately after adding).
This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-wait` flag.
Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
Default: `false`
Type: `flag`
### `Import.BatchMaxNodes`
The maximum number of nodes in a write-batch. The total size of the batch is limited by `BatchMaxnodes` and `BatchMaxSize`.

View File

@ -115,7 +115,7 @@ require (
github.com/libp2p/go-doh-resolver v0.5.0 // indirect
github.com/libp2p/go-flow-metrics v0.3.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb // indirect
github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.14.2 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
@ -171,7 +171,7 @@ require (
github.com/pion/webrtc/v4 v4.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/probe-lab/go-libdht v0.3.0 // indirect
github.com/probe-lab/go-libdht v0.4.0 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect

View File

@ -430,8 +430,8 @@ github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl9
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
@ -630,8 +630,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
github.com/probe-lab/go-libdht v0.3.0 h1:Q3ZXK8wCjZvgeHSTtRrppXobXY/KHPLZJfc+cdTTyqA=
github.com/probe-lab/go-libdht v0.3.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
github.com/probe-lab/go-libdht v0.4.0 h1:LAqHuko/owRW6+0cs5wmJXbHzg09EUMJEh5DI37yXqo=
github.com/probe-lab/go-libdht v0.4.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=

View File

@ -59,7 +59,7 @@ Metrics for the legacy provider system when `Provide.DHT.SweepEnabled=false`:
Metrics for the DHT provider system when `Provide.DHT.SweepEnabled=true`:
- `total_provide_count_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`)
- `provider_provides_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`)
> [!NOTE]
> These metrics are exposed by [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/). You can enable debug logging for DHT provider activity with `GOLOG_LOG_LEVEL=dht/provider=debug`.

View File

@ -47,25 +47,51 @@ Or in your IPFS config file:
The telemetry plugin collects the following anonymized data:
### General Information
- **Agent version**: The version of Kubo being used.
- **Platform details**: Operating system, architecture, and container status.
- **Uptime**: How long the node has been running, categorized into buckets.
- **Repo size**: Categorized into buckets (e.g., 1GB, 5GB, 10GB, etc.).
- **UUID**: Anonymous identifier for this node
- **Agent version**: Kubo version string
- **Private network**: Whether running in a private IPFS network
- **Repository size**: Categorized into privacy-preserving buckets (1GB, 5GB, 10GB, 100GB, 500GB, 1TB, 10TB, >10TB)
- **Uptime**: Categorized into privacy-preserving buckets (1d, 2d, 3d, 7d, 14d, 30d, >30d)
### Routing & Discovery
- **Custom bootstrap peers**: Whether custom `Bootstrap` peers are configured
- **Routing type**: The `Routing.Type` configured for the node
- **Accelerated DHT client**: Whether `Routing.AcceleratedDHTClient` is enabled
- **Delegated routing count**: Number of `Routing.DelegatedRouters` configured
- **AutoConf enabled**: Whether `AutoConf.Enabled` is set
- **Custom AutoConf URL**: Whether custom `AutoConf.URL` is configured
- **mDNS**: Whether `Discovery.MDNS.Enabled` is set
### Content Providing
- **Provide and Reprovide strategy**: The `Provide.Strategy` configured
- **Sweep-based provider**: Whether `Provide.DHT.SweepEnabled` is set
- **Custom Interval**: Whether custom `Provide.DHT.Interval` is configured
- **Custom MaxWorkers**: Whether custom `Provide.DHT.MaxWorkers` is configured
### Network Configuration
- **Private network**: Whether the node is running in a private network.
- **Bootstrap peers**: Whether custom bootstrap peers are used.
- **Routing type**: Whether the node uses DHT, IPFS, or a custom routing setup.
- **AutoNAT settings**: Whether AutoNAT is enabled and its reachability status.
- **AutoConf settings**: Whether AutoConf is enabled and whether a custom URL is used.
- **Swarm settings**: Whether hole punching is enabled, and whether public IP addresses are used.
### TLS and Discovery
- **AutoTLS settings**: Whether WSS is enabled and whether a custom domain suffix is used.
- **Discovery settings**: Whether mDNS is enabled.
- **AutoNAT service mode**: The `AutoNAT.ServiceMode` configured
- **AutoNAT reachability**: Current reachability status determined by AutoNAT
- **Hole punching**: Whether `Swarm.EnableHolePunching` is enabled
- **Circuit relay addresses**: Whether the node advertises circuit relay addresses
- **Public IPv4 addresses**: Whether the node has public IPv4 addresses
- **Public IPv6 addresses**: Whether the node has public IPv6 addresses
- **AutoWSS**: Whether `AutoTLS.AutoWSS` is enabled
- **Custom domain suffix**: Whether custom `AutoTLS.DomainSuffix` is configured
### Reprovider Strategy
- The strategy used for reprovider (e.g., "all", "pinned"...).
### Platform Information
- **Operating system**: The OS the node is running on
- **CPU architecture**: The architecture the node is running on
- **Container detection**: Whether the node is running inside a container
- **VM detection**: Whether the node is running inside a virtual machine
### Code Reference
Data is organized in the `LogEvent` struct at [`plugin/plugins/telemetry/telemetry.go`](https://github.com/ipfs/kubo/blob/master/plugin/plugins/telemetry/telemetry.go). This struct is the authoritative source of truth for all telemetry data, including privacy-preserving buckets for repository size and uptime. Note that this documentation may not always be up-to-date - refer to the code for the current implementation.
---

4
go.mod
View File

@ -53,7 +53,7 @@ require (
github.com/libp2p/go-doh-resolver v0.5.0
github.com/libp2p/go-libp2p v0.45.0
github.com/libp2p/go-libp2p-http v0.5.0
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb
github.com/libp2p/go-libp2p-kbucket v0.8.0
github.com/libp2p/go-libp2p-pubsub v0.14.2
github.com/libp2p/go-libp2p-pubsub-router v0.6.0
@ -69,7 +69,7 @@ require (
github.com/multiformats/go-multihash v0.2.3
github.com/opentracing/opentracing-go v1.2.0
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/probe-lab/go-libdht v0.3.0
github.com/probe-lab/go-libdht v0.4.0
github.com/prometheus/client_golang v1.23.2
github.com/stretchr/testify v1.11.1
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d

8
go.sum
View File

@ -514,8 +514,8 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk
github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA=
github.com/libp2p/go-libp2p-http v0.5.0 h1:+x0AbLaUuLBArHubbbNRTsgWz0RjNTy6DJLOxQ3/QBc=
github.com/libp2p/go-libp2p-http v0.5.0/go.mod h1:glh87nZ35XCQyFsdzZps6+F4HYI6DctVFY5u1fehwSg=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
@ -732,8 +732,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
github.com/probe-lab/go-libdht v0.3.0 h1:Q3ZXK8wCjZvgeHSTtRrppXobXY/KHPLZJfc+cdTTyqA=
github.com/probe-lab/go-libdht v0.3.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
github.com/probe-lab/go-libdht v0.4.0 h1:LAqHuko/owRW6+0cs5wmJXbHzg09EUMJEh5DI37yXqo=
github.com/probe-lab/go-libdht v0.4.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

View File

@ -78,6 +78,7 @@ var uptimeBuckets = []time.Duration{
}
// A LogEvent is the object sent to the telemetry endpoint.
// See https://github.com/ipfs/kubo/blob/master/docs/telemetry.md for details.
type LogEvent struct {
UUID string `json:"uuid"`
@ -91,7 +92,10 @@ type LogEvent struct {
UptimeBucket time.Duration `json:"uptime_bucket"`
ReproviderStrategy string `json:"reprovider_strategy"`
ReproviderStrategy string `json:"reprovider_strategy"`
ProvideDHTSweepEnabled bool `json:"provide_dht_sweep_enabled"`
ProvideDHTIntervalCustom bool `json:"provide_dht_interval_custom"`
ProvideDHTMaxWorkersCustom bool `json:"provide_dht_max_workers_custom"`
RoutingType string `json:"routing_type"`
RoutingAcceleratedDHTClient bool `json:"routing_accelerated_dht_client"`
@ -352,6 +356,7 @@ func (p *telemetryPlugin) Start(n *core.IpfsNode) error {
func (p *telemetryPlugin) prepareEvent() {
p.collectBasicInfo()
p.collectRoutingInfo()
p.collectProvideInfo()
p.collectAutoNATInfo()
p.collectAutoConfInfo()
p.collectSwarmInfo()
@ -360,13 +365,6 @@ func (p *telemetryPlugin) prepareEvent() {
p.collectPlatformInfo()
}
// Collects:
// * AgentVersion
// * PrivateNetwork
// * RepoSizeBucket
// * BootstrappersCustom
// * UptimeBucket
// * ReproviderStrategy
func (p *telemetryPlugin) collectBasicInfo() {
p.event.AgentVersion = ipfs.GetUserAgentVersion()
@ -406,8 +404,6 @@ func (p *telemetryPlugin) collectBasicInfo() {
break
}
p.event.UptimeBucket = uptimeBucket
p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
}
func (p *telemetryPlugin) collectRoutingInfo() {
@ -416,6 +412,13 @@ func (p *telemetryPlugin) collectRoutingInfo() {
p.event.RoutingDelegatedCount = len(p.config.Routing.DelegatedRouters)
}
func (p *telemetryPlugin) collectProvideInfo() {
p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
p.event.ProvideDHTSweepEnabled = p.config.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled)
p.event.ProvideDHTIntervalCustom = !p.config.Provide.DHT.Interval.IsDefault()
p.event.ProvideDHTMaxWorkersCustom = !p.config.Provide.DHT.MaxWorkers.IsDefault()
}
type reachabilityHost interface {
Reachability() network.Reachability
}

View File

@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"
"github.com/dustin/go-humanize"
"github.com/ipfs/kubo/config"
@ -15,6 +16,19 @@ import (
"github.com/stretchr/testify/require"
)
// waitForLogMessage polls a buffer for a log message, waiting up to timeout duration.
// Returns true if message found, false if timeout reached.
func waitForLogMessage(buffer *harness.Buffer, message string, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if strings.Contains(buffer.String(), message) {
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
func TestAdd(t *testing.T) {
t.Parallel()
@ -435,7 +449,182 @@ func TestAdd(t *testing.T) {
require.Equal(t, 992, len(root.Links))
})
})
}
func TestAddFastProvide(t *testing.T) {
t.Parallel()
const (
shortString = "hello world"
shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256
)
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
})
// Start daemon with debug logging
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV0, cidStr)
// Verify fast-provide-root was disabled
daemonLog := node.Daemon.Stderr.String()
require.Contains(t, daemonLog, "fast-provide-root: skipped")
})
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Use default config (FastProvideRoot=true, FastProvideWait=false)
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV0, cidStr)
daemonLog := node.Daemon.Stderr
// Should see async mode started
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
// Wait for async completion or failure (up to 11 seconds - slightly more than fastProvideTimeout)
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", 11*time.Second) ||
waitForLogMessage(daemonLog, "async provide failed", 11*time.Second)
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
})
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideWait = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Use Runner.Run with stdin to allow for expected errors
res := node.Runner.Run(harness.RunRequest{
Path: node.IPFSBin,
Args: []string{"add", "-q"},
CmdOpts: []harness.CmdOpt{
harness.RunWithStdin(strings.NewReader(shortString)),
},
})
// In sync mode (wait=true), provide errors propagate and fail the command.
// Test environment uses 'test' profile with no bootstrappers, and CI has
// insufficient peers for proper DHT puts, so we expect this to fail with
// "failed to find any peer in table" error from the DHT.
require.Equal(t, 1, res.ExitCode())
require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table")
daemonLog := node.Daemon.Stderr.String()
// Should see sync mode started
require.Contains(t, daemonLog, "fast-provide-root: enabled")
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged
})
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
cfg.Import.FastProvideWait = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV0, cidStr)
daemonLog := node.Daemon.Stderr.String()
require.Contains(t, daemonLog, "fast-provide-root: skipped")
require.Contains(t, daemonLog, "wait-flag-ignored")
})
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=true")
require.Equal(t, shortStringCidV0, cidStr)
daemonLog := node.Daemon.Stderr
// Flag should enable it despite config saying false
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
})
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=false")
require.Equal(t, shortStringCidV0, cidStr)
daemonLog := node.Daemon.Stderr.String()
// Flag should disable it despite config saying true
require.Contains(t, daemonLog, "fast-provide-root: skipped")
})
}
// createDirectoryForHAMT aims to create enough files with long names for the directory block to be close to the UnixFSHAMTDirectorySizeThreshold.

View File

@ -0,0 +1,164 @@
package cli
import (
"strings"
"testing"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/stretchr/testify/assert"
"github.com/tidwall/sjson"
)
func TestConfigSecrets(t *testing.T) {
t.Parallel()
t.Run("Identity.PrivKey protection", func(t *testing.T) {
t.Parallel()
t.Run("Identity.PrivKey is concealed in config show", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Read the actual config file to get the real PrivKey
configFile := node.ReadFile(node.ConfigFile())
assert.Contains(t, configFile, "PrivKey")
// config show should NOT contain the PrivKey
configShow := node.RunIPFS("config", "show").Stdout.String()
assert.NotContains(t, configShow, "PrivKey")
})
t.Run("Identity.PrivKey cannot be read via ipfs config", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Attempting to read Identity.PrivKey should fail
res := node.RunIPFS("config", "Identity.PrivKey")
assert.Equal(t, 1, res.ExitCode())
assert.Contains(t, res.Stderr.String(), "cannot show or change private key")
})
t.Run("Identity.PrivKey cannot be read via ipfs config Identity", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Attempting to read Identity section should fail (it contains PrivKey)
res := node.RunIPFS("config", "Identity")
assert.Equal(t, 1, res.ExitCode())
assert.Contains(t, res.Stderr.String(), "cannot show or change private key")
})
t.Run("Identity.PrivKey cannot be set via config replace", func(t *testing.T) {
t.Parallel()
// Key rotation must be done in offline mode via the dedicated `ipfs key rotate` command.
// This test ensures PrivKey cannot be changed via config replace.
node := harness.NewT(t).NewNode().Init()
configShow := node.RunIPFS("config", "show").Stdout.String()
// Try to inject a PrivKey via config replace
configJSON := MustVal(sjson.Set(configShow, "Identity.PrivKey", "CAASqAkwggSkAgEAAo"))
node.WriteBytes("new-config", []byte(configJSON))
res := node.RunIPFS("config", "replace", "new-config")
assert.Equal(t, 1, res.ExitCode())
assert.Contains(t, res.Stderr.String(), "setting private key")
})
t.Run("Identity.PrivKey is preserved when re-injecting config", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Read the original config file
originalConfig := node.ReadFile(node.ConfigFile())
assert.Contains(t, originalConfig, "PrivKey")
// Extract the PrivKey value for comparison
var origPrivKey string
assert.Contains(t, originalConfig, "PrivKey")
// Simple extraction - find the PrivKey line
for _, line := range strings.Split(originalConfig, "\n") {
if strings.Contains(line, "\"PrivKey\":") {
origPrivKey = line
break
}
}
assert.NotEmpty(t, origPrivKey)
// Get config show output (which should NOT contain PrivKey)
configShow := node.RunIPFS("config", "show").Stdout.String()
assert.NotContains(t, configShow, "PrivKey")
// Re-inject the config via config replace
node.WriteBytes("config-show", []byte(configShow))
node.IPFS("config", "replace", "config-show")
// The PrivKey should still be in the config file
newConfig := node.ReadFile(node.ConfigFile())
assert.Contains(t, newConfig, "PrivKey")
// Verify the PrivKey line is the same
var newPrivKey string
for _, line := range strings.Split(newConfig, "\n") {
if strings.Contains(line, "\"PrivKey\":") {
newPrivKey = line
break
}
}
assert.Equal(t, origPrivKey, newPrivKey, "PrivKey should be preserved")
})
})
t.Run("TLS security validation", func(t *testing.T) {
t.Parallel()
t.Run("AutoConf.TLSInsecureSkipVerify defaults to false", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Check the default value in a fresh init
res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify")
// Field may not exist (exit code 1) or be false/empty (exit code 0)
// Both are acceptable as they mean "not true"
output := res.Stdout.String()
assert.NotContains(t, output, "true", "default should not be true")
})
t.Run("AutoConf.TLSInsecureSkipVerify can be set to true", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Set to true
node.IPFS("config", "AutoConf.TLSInsecureSkipVerify", "true", "--json")
// Verify it was set
res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify")
assert.Equal(t, 0, res.ExitCode())
assert.Contains(t, res.Stdout.String(), "true")
})
t.Run("HTTPRetrieval.TLSInsecureSkipVerify defaults to false", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Check the default value in a fresh init
res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify")
// Field may not exist (exit code 1) or be false/empty (exit code 0)
// Both are acceptable as they mean "not true"
output := res.Stdout.String()
assert.NotContains(t, output, "true", "default should not be true")
})
t.Run("HTTPRetrieval.TLSInsecureSkipVerify can be set to true", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Set to true
node.IPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify", "true", "--json")
// Verify it was set
res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify")
assert.Equal(t, 0, res.ExitCode())
assert.Contains(t, res.Stdout.String(), "true")
})
})
}

View File

@ -5,10 +5,13 @@ import (
"io"
"os"
"testing"
"time"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/ipfs/kubo/test/cli/testutils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
@ -102,3 +105,200 @@ func TestDag(t *testing.T) {
assert.Equal(t, content, stat.Stdout.Bytes())
})
}
func TestDagImportFastProvide(t *testing.T) {
t.Parallel()
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
})
// Start daemon with debug logging
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
err = node.IPFSDagImport(r, fixtureCid)
require.NoError(t, err)
// Verify fast-provide-root was disabled
daemonLog := node.Daemon.Stderr.String()
require.Contains(t, daemonLog, "fast-provide-root: skipped")
})
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Use default config (FastProvideRoot=true, FastProvideWait=false)
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
err = node.IPFSDagImport(r, fixtureCid)
require.NoError(t, err)
daemonLog := node.Daemon.Stderr
// Should see async mode started
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
// Wait for async completion or failure (slightly more than DefaultFastProvideTimeout)
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
timeout := config.DefaultFastProvideTimeout + time.Second
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", timeout) ||
waitForLogMessage(daemonLog, "async provide failed", timeout)
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
})
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideWait = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file - use Run instead of IPFSDagImport to handle expected error
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
res := node.Runner.Run(harness.RunRequest{
Path: node.IPFSBin,
Args: []string{"dag", "import", "--pin-roots=false"},
CmdOpts: []harness.CmdOpt{
harness.RunWithStdin(r),
},
})
// In sync mode (wait=true), provide errors propagate and fail the command.
// Test environment uses 'test' profile with no bootstrappers, and CI has
// insufficient peers for proper DHT puts, so we expect this to fail with
// "failed to find any peer in table" error from the DHT.
require.Equal(t, 1, res.ExitCode())
require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table")
daemonLog := node.Daemon.Stderr.String()
// Should see sync mode started
require.Contains(t, daemonLog, "fast-provide-root: enabled")
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
require.Contains(t, daemonLog, fixtureCid) // Should log the specific CID being provided
require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged
})
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
cfg.Import.FastProvideWait = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
err = node.IPFSDagImport(r, fixtureCid)
require.NoError(t, err)
daemonLog := node.Daemon.Stderr.String()
require.Contains(t, daemonLog, "fast-provide-root: skipped")
// Note: dag import doesn't log wait-flag-ignored like add does
})
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.False
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file with flag override
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=true")
require.NoError(t, err)
daemonLog := node.Daemon.Stderr
// Flag should enable it despite config saying false
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
})
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.FastProvideRoot = config.True
})
node.StartDaemonWithReq(harness.RunRequest{
CmdOpts: []harness.CmdOpt{
harness.RunWithEnv(map[string]string{
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
}),
},
}, "")
defer node.StopDaemon()
// Import CAR file with flag override
r, err := os.Open(fixtureFile)
require.NoError(t, err)
defer r.Close()
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=false")
require.NoError(t, err)
daemonLog := node.Daemon.Stderr.String()
// Flag should disable it despite config saying true
require.Contains(t, daemonLog, "fast-provide-root: skipped")
})
}

View File

@ -72,9 +72,9 @@ func TestRoutingV1Proxy(t *testing.T) {
cidStr := nodes[0].IPFSAddStr(string(random.Bytes(1000)))
// Reprovide as initialProviderDelay still ongoing
res := nodes[0].IPFS("routing", "reprovide")
require.NoError(t, res.Err)
res = nodes[1].IPFS("routing", "findprovs", cidStr)
waitUntilProvidesComplete(t, nodes[0])
res := nodes[1].IPFS("routing", "findprovs", cidStr)
assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed())
})

View File

@ -18,7 +18,6 @@ import (
"github.com/ipfs/kubo/test/cli/harness"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// swarmPeersOutput is used to parse the JSON output of 'ipfs swarm peers --enc=json'
@ -48,9 +47,7 @@ func TestRoutingV1Server(t *testing.T) {
text := "hello world " + uuid.New().String()
cidStr := nodes[2].IPFSAddStr(text)
_ = nodes[3].IPFSAddStr(text)
// Reprovide as initialProviderDelay still ongoing
res := nodes[3].IPFS("routing", "reprovide")
require.NoError(t, res.Err)
waitUntilProvidesComplete(t, nodes[3])
cid, err := cid.Decode(cidStr)
assert.NoError(t, err)

View File

@ -17,6 +17,8 @@ func TestDHTOptimisticProvide(t *testing.T) {
nodes[0].UpdateConfig(func(cfg *config.Config) {
cfg.Experimental.OptimisticProvide = true
// Optimistic provide only works with the legacy provider.
cfg.Provide.DHT.SweepEnabled = config.False
})
nodes.StartDaemons().Connect()

View File

@ -0,0 +1,384 @@
package cli
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Well-known block file names in flatfs blockstore that should not be corrupted during testing.
// Flatfs stores each block as a separate .data file on disk.
const (
// emptyFileFlatfsFilename is the flatfs filename for an empty UnixFS file block
emptyFileFlatfsFilename = "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ"
// emptyDirFlatfsFilename is the flatfs filename for an empty UnixFS directory block.
// This block has special handling and may be served from memory even when corrupted on disk.
emptyDirFlatfsFilename = "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"
)
// getEligibleFlatfsBlockFiles returns flatfs block files (*.data) that are safe to corrupt in tests.
// Filters out well-known blocks (empty file/dir) that cause test flakiness.
//
// Note: This helper is specific to the flatfs blockstore implementation where each block
// is stored as a separate file on disk under blocks/*/*.data.
func getEligibleFlatfsBlockFiles(t *testing.T, node *harness.Node) []string {
blockFiles, err := filepath.Glob(filepath.Join(node.Dir, "blocks", "*", "*.data"))
require.NoError(t, err)
require.NotEmpty(t, blockFiles, "no flatfs block files found")
var eligible []string
for _, f := range blockFiles {
name := filepath.Base(f)
if !strings.Contains(name, emptyFileFlatfsFilename) &&
!strings.Contains(name, emptyDirFlatfsFilename) {
eligible = append(eligible, f)
}
}
return eligible
}
// corruptRandomBlock corrupts a random block file in the flatfs blockstore.
// Returns the path to the corrupted file.
func corruptRandomBlock(t *testing.T, node *harness.Node) string {
eligible := getEligibleFlatfsBlockFiles(t, node)
require.NotEmpty(t, eligible, "no eligible blocks to corrupt")
toCorrupt := eligible[0]
err := os.WriteFile(toCorrupt, []byte("corrupted data"), 0644)
require.NoError(t, err)
return toCorrupt
}
// corruptMultipleBlocks corrupts multiple block files in the flatfs blockstore.
// Returns the paths to the corrupted files.
func corruptMultipleBlocks(t *testing.T, node *harness.Node, count int) []string {
eligible := getEligibleFlatfsBlockFiles(t, node)
require.GreaterOrEqual(t, len(eligible), count, "not enough eligible blocks to corrupt")
var corrupted []string
for i := 0; i < count && i < len(eligible); i++ {
err := os.WriteFile(eligible[i], []byte(fmt.Sprintf("corrupted data %d", i)), 0644)
require.NoError(t, err)
corrupted = append(corrupted, eligible[i])
}
return corrupted
}
func TestRepoVerify(t *testing.T) {
t.Run("healthy repo passes", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.IPFS("add", "-q", "--raw-leaves=false", "-r", node.IPFSBin)
res := node.IPFS("repo", "verify")
assert.Contains(t, res.Stdout.String(), "all blocks validated")
})
t.Run("detects corruption", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.IPFSAddStr("test content")
corruptRandomBlock(t, node)
res := node.RunIPFS("repo", "verify")
assert.Equal(t, 1, res.ExitCode())
assert.Contains(t, res.Stdout.String(), "was corrupt")
assert.Contains(t, res.Stderr.String(), "1 blocks corrupt")
})
t.Run("drop removes corrupt blocks", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
cid := node.IPFSAddStr("test content")
corruptRandomBlock(t, node)
res := node.RunIPFS("repo", "verify", "--drop")
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
output := res.Stdout.String()
assert.Contains(t, output, "1 blocks corrupt")
assert.Contains(t, output, "1 removed")
// Verify block is gone
res = node.RunIPFS("block", "stat", cid)
assert.NotEqual(t, 0, res.ExitCode())
})
t.Run("heal requires online mode", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.IPFSAddStr("test content")
corruptRandomBlock(t, node)
res := node.RunIPFS("repo", "verify", "--heal")
assert.NotEqual(t, 0, res.ExitCode())
assert.Contains(t, res.Stderr.String(), "online mode")
})
t.Run("heal repairs from network", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(2).Init()
nodes.StartDaemons().Connect()
defer nodes.StopDaemons()
// Add content to node 0
cid := nodes[0].IPFSAddStr("test content for healing")
// Wait for it to appear on node 1
nodes[1].IPFS("block", "get", cid)
// Corrupt on node 1
corruptRandomBlock(t, nodes[1])
// Heal should restore from node 0
res := nodes[1].RunIPFS("repo", "verify", "--heal")
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
output := res.Stdout.String()
// Should report corruption and healing with specific counts
assert.Contains(t, output, "1 blocks corrupt")
assert.Contains(t, output, "1 removed")
assert.Contains(t, output, "1 healed")
// Verify block is restored
nodes[1].IPFS("block", "stat", cid)
})
t.Run("healed blocks contain correct data", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(2).Init()
nodes.StartDaemons().Connect()
defer nodes.StopDaemons()
// Add specific content to node 0
testContent := "this is the exact content that should be healed correctly"
cid := nodes[0].IPFSAddStr(testContent)
// Fetch to node 1 and verify the content is correct initially
nodes[1].IPFS("block", "get", cid)
res := nodes[1].IPFS("cat", cid)
assert.Equal(t, testContent, res.Stdout.String())
// Corrupt on node 1
corruptRandomBlock(t, nodes[1])
// Heal the corruption
res = nodes[1].RunIPFS("repo", "verify", "--heal")
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
output := res.Stdout.String()
assert.Contains(t, output, "1 blocks corrupt")
assert.Contains(t, output, "1 removed")
assert.Contains(t, output, "1 healed")
// Verify the healed content matches the original exactly
res = nodes[1].IPFS("cat", cid)
assert.Equal(t, testContent, res.Stdout.String(), "healed content should match original")
// Also verify via block get that the raw block data is correct
block0 := nodes[0].IPFS("block", "get", cid)
block1 := nodes[1].IPFS("block", "get", cid)
assert.Equal(t, block0.Stdout.String(), block1.Stdout.String(), "raw block data should match")
})
t.Run("multiple corrupt blocks", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Create 20 blocks
for i := 0; i < 20; i++ {
node.IPFSAddStr(strings.Repeat("test content ", i+1))
}
// Corrupt 5 blocks
corruptMultipleBlocks(t, node, 5)
// Verify detects all corruptions
res := node.RunIPFS("repo", "verify")
assert.Equal(t, 1, res.ExitCode())
// Error summary is in stderr
assert.Contains(t, res.Stderr.String(), "5 blocks corrupt")
// Test with --drop
res = node.RunIPFS("repo", "verify", "--drop")
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
assert.Contains(t, res.Stdout.String(), "5 blocks corrupt")
assert.Contains(t, res.Stdout.String(), "5 removed")
})
t.Run("empty repository", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Verify empty repo passes
res := node.IPFS("repo", "verify")
assert.Equal(t, 0, res.ExitCode())
assert.Contains(t, res.Stdout.String(), "all blocks validated")
// Should work with --drop and --heal too
res = node.IPFS("repo", "verify", "--drop")
assert.Equal(t, 0, res.ExitCode())
assert.Contains(t, res.Stdout.String(), "all blocks validated")
})
t.Run("partial heal success", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(2).Init()
// Start both nodes and connect them
nodes.StartDaemons().Connect()
defer nodes.StopDaemons()
// Add 5 blocks to node 0, pin them to keep available
cid1 := nodes[0].IPFSAddStr("content available for healing 1")
cid2 := nodes[0].IPFSAddStr("content available for healing 2")
cid3 := nodes[0].IPFSAddStr("content available for healing 3")
cid4 := nodes[0].IPFSAddStr("content available for healing 4")
cid5 := nodes[0].IPFSAddStr("content available for healing 5")
// Pin these on node 0 to ensure they stay available
nodes[0].IPFS("pin", "add", cid1)
nodes[0].IPFS("pin", "add", cid2)
nodes[0].IPFS("pin", "add", cid3)
nodes[0].IPFS("pin", "add", cid4)
nodes[0].IPFS("pin", "add", cid5)
// Node 1 fetches these blocks
nodes[1].IPFS("block", "get", cid1)
nodes[1].IPFS("block", "get", cid2)
nodes[1].IPFS("block", "get", cid3)
nodes[1].IPFS("block", "get", cid4)
nodes[1].IPFS("block", "get", cid5)
// Now remove some blocks from node 0 to simulate partial availability
nodes[0].IPFS("pin", "rm", cid3)
nodes[0].IPFS("pin", "rm", cid4)
nodes[0].IPFS("pin", "rm", cid5)
nodes[0].IPFS("repo", "gc")
// Verify node 1 is still connected
peers := nodes[1].IPFS("swarm", "peers")
require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
// Corrupt 5 blocks on node 1
corruptMultipleBlocks(t, nodes[1], 5)
// Heal should partially succeed (only cid1 and cid2 available from node 0)
res := nodes[1].RunIPFS("repo", "verify", "--heal")
assert.Equal(t, 1, res.ExitCode())
// Should show mixed results with specific counts in stderr
errOutput := res.Stderr.String()
assert.Contains(t, errOutput, "5 blocks corrupt")
assert.Contains(t, errOutput, "5 removed")
// Only cid1 and cid2 are available for healing, cid3-5 were GC'd
assert.Contains(t, errOutput, "2 healed")
assert.Contains(t, errOutput, "3 failed to heal")
})
t.Run("heal with block not available on network", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(2).Init()
// Start both nodes and connect
nodes.StartDaemons().Connect()
defer nodes.StopDaemons()
// Add unique content only to node 1
nodes[1].IPFSAddStr("unique content that exists nowhere else")
// Ensure nodes are connected
peers := nodes[1].IPFS("swarm", "peers")
require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
// Corrupt the block on node 1
corruptRandomBlock(t, nodes[1])
// Heal should fail - node 0 doesn't have this content
res := nodes[1].RunIPFS("repo", "verify", "--heal")
assert.Equal(t, 1, res.ExitCode())
// Should report heal failure with specific counts in stderr
errOutput := res.Stderr.String()
assert.Contains(t, errOutput, "1 blocks corrupt")
assert.Contains(t, errOutput, "1 removed")
assert.Contains(t, errOutput, "1 failed to heal")
})
t.Run("large repository scale test", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Create 1000 small blocks
for i := 0; i < 1000; i++ {
node.IPFSAddStr(fmt.Sprintf("content-%d", i))
}
// Corrupt 10 blocks
corruptMultipleBlocks(t, node, 10)
// Verify handles large repos efficiently
res := node.RunIPFS("repo", "verify")
assert.Equal(t, 1, res.ExitCode())
// Should report exactly 10 corrupt blocks in stderr
assert.Contains(t, res.Stderr.String(), "10 blocks corrupt")
// Test --drop at scale
res = node.RunIPFS("repo", "verify", "--drop")
assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
output := res.Stdout.String()
assert.Contains(t, output, "10 blocks corrupt")
assert.Contains(t, output, "10 removed")
})
t.Run("drop with partial removal failures", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
// Create several blocks
for i := 0; i < 5; i++ {
node.IPFSAddStr(fmt.Sprintf("content for removal test %d", i))
}
// Corrupt 3 blocks
corruptedFiles := corruptMultipleBlocks(t, node, 3)
require.Len(t, corruptedFiles, 3)
// Make one of the corrupted files read-only to simulate removal failure
err := os.Chmod(corruptedFiles[0], 0400) // read-only
require.NoError(t, err)
defer func() { _ = os.Chmod(corruptedFiles[0], 0644) }() // cleanup
// Also make the directory read-only to prevent deletion
blockDir := filepath.Dir(corruptedFiles[0])
originalPerm, err := os.Stat(blockDir)
require.NoError(t, err)
err = os.Chmod(blockDir, 0500) // read+execute only, no write
require.NoError(t, err)
defer func() { _ = os.Chmod(blockDir, originalPerm.Mode()) }() // cleanup
// Try to drop - should fail because at least one block can't be removed
res := node.RunIPFS("repo", "verify", "--drop")
assert.Equal(t, 1, res.ExitCode(), "should exit 1 when some blocks fail to remove")
// Restore permissions for verification
_ = os.Chmod(blockDir, originalPerm.Mode())
_ = os.Chmod(corruptedFiles[0], 0644)
// Should report both successes and failures with specific counts
errOutput := res.Stderr.String()
assert.Contains(t, errOutput, "3 blocks corrupt")
assert.Contains(t, errOutput, "2 removed")
assert.Contains(t, errOutput, "1 failed to remove")
})
}

View File

@ -2,7 +2,10 @@ package cli
import (
"fmt"
"strconv"
"strings"
"testing"
"time"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/ipfs/kubo/test/cli/testutils"
@ -10,6 +13,33 @@ import (
"github.com/stretchr/testify/require"
)
func waitUntilProvidesComplete(t *testing.T, n *harness.Node) {
getCidsCount := func(line string) int {
trimmed := strings.TrimSpace(line)
countStr := strings.SplitN(trimmed, " ", 2)[0]
count, err := strconv.Atoi(countStr)
require.NoError(t, err)
return count
}
queuedProvides, ongoingProvides := true, true
for queuedProvides || ongoingProvides {
res := n.IPFS("provide", "stat", "-a")
require.NoError(t, res.Err)
for _, line := range res.Stdout.Lines() {
if trimmed, ok := strings.CutPrefix(line, " Provide queue:"); ok {
provideQueueSize := getCidsCount(trimmed)
queuedProvides = provideQueueSize > 0
}
if trimmed, ok := strings.CutPrefix(line, " Ongoing provides:"); ok {
ongoingProvideCount := getCidsCount(trimmed)
ongoingProvides = ongoingProvideCount > 0
}
}
time.Sleep(10 * time.Millisecond)
}
}
func testRoutingDHT(t *testing.T, enablePubsub bool) {
t.Run(fmt.Sprintf("enablePubSub=%v", enablePubsub), func(t *testing.T) {
t.Parallel()
@ -84,10 +114,8 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
t.Run("ipfs routing findprovs", func(t *testing.T) {
t.Parallel()
hash := nodes[3].IPFSAddStr("some stuff")
// Reprovide as initialProviderDelay still ongoing
res := nodes[3].IPFS("routing", "reprovide")
require.NoError(t, res.Err)
res = nodes[4].IPFS("routing", "findprovs", hash)
waitUntilProvidesComplete(t, nodes[3])
res := nodes[4].IPFS("routing", "findprovs", hash)
assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed())
})

View File

@ -159,4 +159,127 @@ func TestRPCAuth(t *testing.T) {
node.StopDaemon()
})
t.Run("Requests without Authorization header are rejected when auth is enabled", func(t *testing.T) {
t.Parallel()
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
"userA": {
AuthSecret: "bearer:mytoken",
AllowedPaths: []string{"/api/v0"},
},
})
// Create client with NO auth
apiClient := node.APIClient() // Uses http.DefaultClient with no auth headers
// Should be denied without auth header
resp := apiClient.Post("/api/v0/id", nil)
assert.Equal(t, 403, resp.StatusCode)
// Should contain denial message
assert.Contains(t, resp.Body, rpcDeniedMsg)
node.StopDaemon()
})
t.Run("Version endpoint is always accessible even with limited AllowedPaths", func(t *testing.T) {
t.Parallel()
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
"userA": {
AuthSecret: "bearer:mytoken",
AllowedPaths: []string{"/api/v0/id"}, // Only /id allowed
},
})
apiClient := node.APIClient()
apiClient.Client = &http.Client{
Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport),
}
// Can access /version even though not in AllowedPaths
resp := apiClient.Post("/api/v0/version", nil)
assert.Equal(t, 200, resp.StatusCode)
node.StopDaemon()
})
t.Run("User cannot access API with another user's secret", func(t *testing.T) {
t.Parallel()
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
"alice": {
AuthSecret: "bearer:alice-secret",
AllowedPaths: []string{"/api/v0/id"},
},
"bob": {
AuthSecret: "bearer:bob-secret",
AllowedPaths: []string{"/api/v0/config"},
},
})
// Alice tries to use Bob's secret
apiClient := node.APIClient()
apiClient.Client = &http.Client{
Transport: auth.NewAuthorizedRoundTripper("Bearer bob-secret", http.DefaultTransport),
}
// Bob's secret should work for Bob's paths
resp := apiClient.Post("/api/v0/config/show", nil)
assert.Equal(t, 200, resp.StatusCode)
// But not for Alice's paths (Bob doesn't have access to /id)
resp = apiClient.Post("/api/v0/id", nil)
assert.Equal(t, 403, resp.StatusCode)
node.StopDaemon()
})
t.Run("Empty AllowedPaths denies all access except version", func(t *testing.T) {
t.Parallel()
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
"userA": {
AuthSecret: "bearer:mytoken",
AllowedPaths: []string{}, // Empty!
},
})
apiClient := node.APIClient()
apiClient.Client = &http.Client{
Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport),
}
// Should deny everything
resp := apiClient.Post("/api/v0/id", nil)
assert.Equal(t, 403, resp.StatusCode)
resp = apiClient.Post("/api/v0/config/show", nil)
assert.Equal(t, 403, resp.StatusCode)
// Except version
resp = apiClient.Post("/api/v0/version", nil)
assert.Equal(t, 200, resp.StatusCode)
node.StopDaemon()
})
t.Run("CLI commands fail without --api-auth when auth is enabled", func(t *testing.T) {
t.Parallel()
node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{
"userA": {
AuthSecret: "bearer:mytoken",
AllowedPaths: []string{"/api/v0"},
},
})
// Try to run command without --api-auth flag
resp := node.RunIPFS("id") // No --api-auth flag
require.Error(t, resp.Err)
require.Contains(t, resp.Stderr.String(), rpcDeniedMsg)
node.StopDaemon()
})
}

View File

@ -205,6 +205,9 @@ func TestTelemetry(t *testing.T) {
"repo_size_bucket",
"uptime_bucket",
"reprovider_strategy",
"provide_dht_sweep_enabled",
"provide_dht_interval_custom",
"provide_dht_max_workers_custom",
"routing_type",
"routing_accelerated_dht_client",
"routing_delegated_count",

View File

@ -184,7 +184,7 @@ require (
github.com/libp2p/go-flow-metrics v0.3.0 // indirect
github.com/libp2p/go-libp2p v0.45.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb // indirect
github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect
github.com/libp2p/go-libp2p-record v0.3.1 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.6-0.20251016083611-f098f492895e // indirect

View File

@ -464,8 +464,8 @@ github.com/libp2p/go-libp2p v0.45.0 h1:Pdhr2HsFXaYjtfiNcBP4CcRUONvbMFdH3puM9vV4T
github.com/libp2p/go-libp2p v0.45.0/go.mod h1:NovCojezAt4dnDd4fH048K7PKEqH0UFYYqJRjIIu8zc=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32 h1:xZj18PsLD157snR/BFo547jwOkGDH7jZjMEkBDOoD4Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251025120456-f33906fd2f32/go.mod h1:aHMTg23iseX9grGSfA5gFUzLrqzmYbA8PqgGPqM8VkI=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb h1:jOWsCSRZKnRgocz4Ocu25Yigh5ZUkar2zWt/bzBh43Q=
github.com/libp2p/go-libp2p-kad-dht v0.35.2-0.20251112013111-6d2d861e0abb/go.mod h1:WIysu8hNWQN8t73dKyTNqiZdcYKRrGFl4wjzX4Gz6pQ=
github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s=
github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4=
github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg=

View File

@ -93,8 +93,8 @@ EOF
test_cmp expected actual
'
test_expect_failure "'ipfs add' with an unregistered hash and wrapped leaves fails without crashing" '
ipfs add --hash poseidon-bls12_381-a2-fc1 --raw-leaves=false -r mountdir/planets
test_expect_success "'ipfs add' with an unregistered hash and wrapped leaves fails without crashing" '
test_expect_code 1 ipfs add --hash poseidon-bls12_381-a2-fc1 --raw-leaves=false -r mountdir/planets
'
}

View File

@ -3,6 +3,9 @@
# Copyright (c) 2016 Jeromy Johnson
# MIT Licensed; see the LICENSE file in this repository.
#
# NOTE: This is a legacy sharness test kept for compatibility.
# New tests for 'ipfs repo verify' should be added to test/cli/repo_verify_test.go
#
test_description="Test ipfs repo fsck"

View File

@ -250,6 +250,5 @@ process_resident_memory_bytes
process_start_time_seconds
process_virtual_memory_bytes
process_virtual_memory_max_bytes
provider_reprovider_provide_count
provider_reprovider_reprovide_count
provider_provides_total
target_info

View File

@ -11,7 +11,7 @@ import (
var CurrentCommit string
// CurrentVersionNumber is the current application's version literal.
const CurrentVersionNumber = "0.39.0-dev"
const CurrentVersionNumber = "0.40.0-dev"
const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint