mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 18:37:45 +08:00
feat(dag): add fast-provide support for dag import
Adds --fast-provide-root and --fast-provide-wait flags to `ipfs dag import`, mirroring the fast-provide functionality available in `ipfs add`. Changes: - Add --fast-provide-root and --fast-provide-wait flags to dag import command - Implement fast-provide logic for all root CIDs in imported CAR files - Works even when --pin-roots=false (strategy checked internally) - Share ExecuteFastProvide implementation between add and dag import - Move ExecuteFastProvide to cmdenv package to avoid import cycles - Add logging when fast-provide is disabled - Conditional error handling: return error when wait=true, warn when wait=false - Update config docs to mention both ipfs add and ipfs dag import - Update changelog to use "provide" terminology and include dag import examples - Add comprehensive test coverage (TestDagImportFastProvide with 6 test cases) The fast-provide feature allows immediate DHT announcement of root CIDs for faster content discovery, bypassing the regular background queue.
This commit is contained in:
parent
f46769efd1
commit
f3a3fceed5
@ -602,12 +602,12 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ExecuteFastProvide(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet)
|
||||
cmdenv.ExecuteFastProvide(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet)
|
||||
} else if !fastProvideRoot {
|
||||
if fastProvideWait {
|
||||
log.Debugw("fast-provide-root: disabled", "wait-flag-ignored", true)
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: disabled")
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,15 +1,19 @@
|
||||
package cmdenv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/kubo/commands"
|
||||
"github.com/ipfs/kubo/core"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
routing "github.com/libp2p/go-libp2p/core/routing"
|
||||
|
||||
"github.com/ipfs/kubo/commands"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/core"
|
||||
coreiface "github.com/ipfs/kubo/core/coreiface"
|
||||
options "github.com/ipfs/kubo/core/coreiface/options"
|
||||
)
|
||||
@ -86,3 +90,96 @@ func needEscape(s string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// provideCIDSync performs a synchronous/blocking provide operation to announce
|
||||
// the given CID to the DHT.
|
||||
//
|
||||
// - If the accelerated DHT client is used, a DHT lookup isn't needed, we
|
||||
// directly allocate provider records to closest peers.
|
||||
// - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an
|
||||
// optimistic provide call.
|
||||
// - Else we make a standard provide call (much slower).
|
||||
//
|
||||
// IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient()
|
||||
// before calling this function. Calling with a nil or invalid router will cause
|
||||
// a panic - this is the caller's responsibility to prevent.
|
||||
func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error {
|
||||
return router.Provide(ctx, c, true)
|
||||
}
|
||||
|
||||
// ExecuteFastProvide immediately provides a root CID to the DHT, bypassing the regular
|
||||
// provide queue for faster content discovery. This function is reusable across commands
|
||||
// that add or import content, such as ipfs add and ipfs dag import.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: context for synchronous provides
|
||||
// - ipfsNode: the IPFS node instance
|
||||
// - cfg: node configuration
|
||||
// - rootCid: the CID to provide
|
||||
// - wait: whether to block until provide completes (sync mode)
|
||||
// - isPinned: whether content is pinned
|
||||
// - isPinnedRoot: whether this is a pinned root CID
|
||||
// - isMFS: whether content is in MFS
|
||||
//
|
||||
// The function handles all precondition checks (Provide.Enabled, DHT availability,
|
||||
// strategy matching) and logs appropriately. In async mode, it launches a goroutine
|
||||
// with a detached context and timeout.
|
||||
func ExecuteFastProvide(
|
||||
ctx context.Context,
|
||||
ipfsNode *core.IpfsNode,
|
||||
cfg *config.Config,
|
||||
rootCid cid.Cid,
|
||||
wait bool,
|
||||
isPinned bool,
|
||||
isPinnedRoot bool,
|
||||
isMFS bool,
|
||||
) {
|
||||
log.Debugw("fast-provide-root: enabled", "wait", wait)
|
||||
|
||||
// Check preconditions for providing
|
||||
switch {
|
||||
case !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled):
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "Provide.Enabled is false")
|
||||
return
|
||||
case cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0:
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "Provide.DHT.Interval is 0")
|
||||
return
|
||||
case !ipfsNode.HasActiveDHTClient():
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "DHT not available")
|
||||
return
|
||||
}
|
||||
|
||||
// Check if strategy allows providing this content
|
||||
strategyStr := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
|
||||
strategy := config.ParseProvideStrategy(strategyStr)
|
||||
shouldProvide := config.ShouldProvideForStrategy(strategy, isPinned, isPinnedRoot, isMFS)
|
||||
|
||||
if !shouldProvide {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "strategy does not match content", "strategy", strategyStr, "pinned", isPinned, "pinnedRoot", isPinnedRoot, "mfs", isMFS)
|
||||
return
|
||||
}
|
||||
|
||||
// Execute provide operation
|
||||
if wait {
|
||||
// Synchronous mode: block until provide completes
|
||||
log.Debugw("fast-provide-root: providing synchronously", "cid", rootCid)
|
||||
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
|
||||
log.Warnw("fast-provide-root: sync provide failed", "cid", rootCid, "error", err)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: sync provide completed", "cid", rootCid)
|
||||
}
|
||||
} else {
|
||||
// Asynchronous mode (default): fire-and-forget, don't block
|
||||
log.Debugw("fast-provide-root: providing asynchronously", "cid", rootCid)
|
||||
go func() {
|
||||
// Use detached context with timeout to prevent hanging on network issues
|
||||
ctx, cancel := context.WithTimeout(context.Background(), config.DefaultFastProvideTimeout)
|
||||
defer cancel()
|
||||
if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil {
|
||||
log.Warnw("fast-provide-root: async provide failed", "cid", rootCid, "error", err)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: async provide completed", "cid", rootCid)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,10 +16,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
pinRootsOptionName = "pin-roots"
|
||||
progressOptionName = "progress"
|
||||
silentOptionName = "silent"
|
||||
statsOptionName = "stats"
|
||||
pinRootsOptionName = "pin-roots"
|
||||
progressOptionName = "progress"
|
||||
silentOptionName = "silent"
|
||||
statsOptionName = "stats"
|
||||
fastProvideRootOptionName = "fast-provide-root"
|
||||
fastProvideWaitOptionName = "fast-provide-wait"
|
||||
)
|
||||
|
||||
// DagCmd provides a subset of commands for interacting with ipld dag objects
|
||||
@ -200,6 +202,8 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
|
||||
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
|
||||
cmds.BoolOption(silentOptionName, "No output."),
|
||||
cmds.BoolOption(statsOptionName, "Output stats."),
|
||||
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
|
||||
cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"),
|
||||
cmdutils.AllowBigBlockOption,
|
||||
},
|
||||
Type: CarImportOutput{},
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
ipldlegacy "github.com/ipfs/go-ipld-legacy"
|
||||
"github.com/ipfs/kubo/config"
|
||||
@ -19,6 +20,8 @@ import (
|
||||
"github.com/ipfs/kubo/core/commands/cmdutils"
|
||||
)
|
||||
|
||||
var log = logging.Logger("core/commands")
|
||||
|
||||
func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
||||
node, err := cmdenv.GetNode(env)
|
||||
if err != nil {
|
||||
@ -47,6 +50,12 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
|
||||
doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
|
||||
|
||||
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
|
||||
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
|
||||
|
||||
fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot)
|
||||
fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait)
|
||||
|
||||
// grab a pinlock ( which doubles as a GC lock ) so that regardless of the
|
||||
// size of the streamed-in cars nothing will disappear on us before we had
|
||||
// a chance to roots that may show up at the very end
|
||||
@ -191,5 +200,25 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
}
|
||||
}
|
||||
|
||||
// Fast-provide roots for faster discovery
|
||||
if fastProvideRoot {
|
||||
err = roots.ForEach(func(c cid.Cid) error {
|
||||
cmdenv.ExecuteFastProvide(req.Context, node, cfg, c, fastProvideWait, doPinRoots, doPinRoots, false)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if fastProvideWait {
|
||||
return err
|
||||
}
|
||||
log.Warnw("fast-provide-root: ForEach error", "error", err)
|
||||
}
|
||||
} else {
|
||||
if fastProvideWait {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true)
|
||||
} else {
|
||||
log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -53,29 +53,36 @@ For background on the sweep provider design and motivations, see [`Provide.DHT.S
|
||||
|
||||
#### ⚡ Fast root CID providing for immediate content discovery
|
||||
|
||||
When you add content to IPFS, it normally gets queued for announcement on the DHT. This background queue can take time to process, meaning other peers won't find your content immediately after `ipfs add` completes.
|
||||
When you add content to IPFS, it normally gets queued for providing on the DHT. This background queue can take time to process, meaning other peers won't find your content immediately after `ipfs add` or `ipfs dag import` completes.
|
||||
|
||||
To make sharing faster, `ipfs add` now does an extra immediate announcement of just the root CID to the DHT (controlled by the new `--fast-provide-root` flag, enabled by default). This lets other peers start discovering your content right away, while the regular background queue still handles announcing all the blocks later.
|
||||
To make sharing faster, `ipfs add` and `ipfs dag import` now do an extra immediate provide of root CIDs to the DHT (controlled by the new `--fast-provide-root` flag, enabled by default). This lets other peers start discovering your content right away, while the regular background queue still handles providing all the blocks later.
|
||||
|
||||
By default, this extra announcement runs in the background without slowing down the command. For use cases requiring guaranteed discoverability before the command returns (for example, sharing a link immediately), use `--fast-provide-wait` to block until the announcement completes.
|
||||
By default, this extra provide runs in the background without slowing down the command. For use cases requiring guaranteed discoverability before the command returns (for example, sharing a link immediately), use `--fast-provide-wait` to block until the provide completes.
|
||||
|
||||
**Configuration:** You can now set default behavior via configuration:
|
||||
|
||||
- `Import.FastProvideRoot` (default: `true`) - Controls whether fast-provide-root is enabled by default for `ipfs add`
|
||||
- `Import.FastProvideWait` (default: `false`) - Controls whether `ipfs add` waits for the announcement to complete by default
|
||||
- `Import.FastProvideRoot` (default: `true`) - Controls whether fast-provide is enabled by default for `ipfs add` and `ipfs dag import`
|
||||
- `Import.FastProvideWait` (default: `false`) - Controls whether commands wait for the provide to complete by default
|
||||
|
||||
These settings follow the same pattern as other Import configuration options (like `Import.CidVersion` or `Import.UnixFSChunker`) and can be overridden on a per-command basis using the `--fast-provide-root` and `--fast-provide-wait` flags.
|
||||
These settings follow the same pattern as other Import configuration options (like `Import.CidVersion` or `Import.UnixFSChunker`) and can be overridden on a per-command basis using the command flags.
|
||||
|
||||
**Usage examples:**
|
||||
|
||||
```bash
|
||||
ipfs add file.txt # Root CID provided immediately in background, independent of queue (default)
|
||||
ipfs add file.txt --fast-provide-wait # Blocks until root CID announcement completes (slower, guaranteed)
|
||||
ipfs add file.txt --fast-provide-root=false # Skip immediate announcement, use background queue only
|
||||
# ipfs add
|
||||
ipfs add file.txt # Root CID provided immediately in background (default)
|
||||
ipfs add file.txt --fast-provide-wait # Blocks until root CID provide completes
|
||||
ipfs add file.txt --fast-provide-root=false # Skip immediate provide, use background queue only
|
||||
|
||||
# ipfs dag import
|
||||
ipfs dag import file.car # Root CIDs provided immediately in background (default)
|
||||
ipfs dag import file.car --fast-provide-wait # Blocks until provides complete
|
||||
ipfs dag import file.car --fast-provide-root=false # Skip immediate provides
|
||||
ipfs dag import file.car --pin-roots=false # No pinning, but provide still works (if strategy allows)
|
||||
|
||||
# Configure default behavior
|
||||
ipfs config --json Import.FastProvideRoot false # Disable fast-provide-root by default
|
||||
ipfs config --json Import.FastProvideWait true # Wait for announcement by default
|
||||
ipfs config --json Import.FastProvideRoot false # Disable fast-provide by default
|
||||
ipfs config --json Import.FastProvideWait true # Wait for provides by default
|
||||
```
|
||||
|
||||
This optimization works best with the sweep provider and accelerated DHT client, where provide operations are significantly faster than traditional DHT providing. The feature is automatically skipped when DHT is unavailable (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
@ -3623,11 +3623,11 @@ Type: `optionalString`
|
||||
|
||||
### `Import.FastProvideRoot`
|
||||
|
||||
Controls the default behavior for whether `ipfs add` immediately provides the root CID to the DHT in addition to the regular provide queue.
|
||||
Controls the default behavior for whether `ipfs add` and `ipfs dag import` immediately provide root CIDs to the DHT in addition to the regular provide queue.
|
||||
|
||||
When enabled (default), the root CID is immediately provided to the DHT, bypassing the regular provide and reprovide queues, allowing other peers to discover your content right away. When disabled, only the regular provide queue is used.
|
||||
When enabled (default), root CIDs are immediately provided to the DHT, bypassing the regular provide and reprovide queues, allowing other peers to discover your content right away. When disabled, only the regular provide queue is used.
|
||||
|
||||
This setting provides the default for the `--fast-provide-root` flag in `ipfs add`. Users can override this default globally by setting a value in the config, or per import operation by passing `ipfs add --fast-provide-root=true` or `ipfs add --fast-provide-root=false`.
|
||||
This setting provides the default for the `--fast-provide-root` flag in `ipfs add` and the `--fast-provide-roots` flag in `ipfs dag import`. Users can override this default globally by setting a value in the config, or per import operation by passing `--fast-provide-root=true`/`false` or `--fast-provide-roots=true`/`false`.
|
||||
|
||||
Note: This flag is ignored if DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
@ -3637,11 +3637,11 @@ Type: `flag`
|
||||
|
||||
### `Import.FastProvideWait`
|
||||
|
||||
Controls the default behavior for whether `ipfs add` blocks and waits for the immediate root CID provide to complete before returning success.
|
||||
Controls the default behavior for whether `ipfs add` and `ipfs dag import` block and wait for the immediate root CID provide to complete before returning success.
|
||||
|
||||
When enabled, the command blocks until the immediate provide completes, ensuring guaranteed discoverability. When disabled (default), the immediate provide happens asynchronously in the background, still making the root CID resolvable via DHT routing faster than the regular provide queue would.
|
||||
When enabled, the command blocks until the immediate provide completes, ensuring guaranteed discoverability. When disabled (default), the immediate provide happens asynchronously in the background, still making root CIDs resolvable via DHT routing faster than the regular provide queue would.
|
||||
|
||||
This setting provides the default for the `--fast-provide-wait` flag in `ipfs add`. Users can override this default globally by setting a value in the config, or per import operation by passing `ipfs add --fast-provide-wait=true` or `ipfs add --fast-provide-wait=false`.
|
||||
This setting provides the default for the `--fast-provide-wait` flag in both `ipfs add` and `ipfs dag import`. Users can override this default globally by setting a value in the config, or per import operation by passing `--fast-provide-wait=true` or `--fast-provide-wait=false`.
|
||||
|
||||
Note: This flag is ignored if DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations).
|
||||
|
||||
|
||||
@ -470,7 +470,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -481,7 +481,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
|
||||
// Verify fast-provide-root was disabled
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: disabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
|
||||
@ -492,7 +492,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -523,7 +523,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -553,7 +553,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -563,7 +563,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
require.Equal(t, shortStringCidV0, cidStr)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: disabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
require.Contains(t, daemonLog, "wait-flag-ignored")
|
||||
})
|
||||
|
||||
@ -577,7 +577,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -602,7 +602,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug",
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
@ -613,7 +613,7 @@ func TestAddFastProvide(t *testing.T) {
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Flag should disable it despite config saying true
|
||||
require.Contains(t, daemonLog, "fast-provide-root: disabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -4,11 +4,15 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -102,3 +106,192 @@ func TestDag(t *testing.T) {
|
||||
assert.Equal(t, content, stat.Stdout.Bytes())
|
||||
})
|
||||
}
|
||||
|
||||
func TestDagImportFastProvide(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
// Start daemon with debug logging
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify fast-provide-root was disabled
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
// Use default config (FastProvideRoot=true, FastProvideWait=false)
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Should see async mode started
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
|
||||
|
||||
// Wait for async completion or failure (slightly more than DefaultFastProvideTimeout)
|
||||
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
|
||||
timeout := config.DefaultFastProvideTimeout + time.Second
|
||||
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", timeout) ||
|
||||
waitForLogMessage(daemonLog, "async provide failed", timeout)
|
||||
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Should see sync mode started
|
||||
require.Contains(t, daemonLog, "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
|
||||
require.Contains(t, daemonLog, fixtureCid) // Should log the specific CID being provided
|
||||
// In test environment with no DHT peers, this will fail, but the provide attempt was made
|
||||
require.True(t,
|
||||
strings.Contains(daemonLog, "sync provide completed") || strings.Contains(daemonLog, "sync provide failed"),
|
||||
"sync provide should complete or fail")
|
||||
})
|
||||
|
||||
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
cfg.Import.FastProvideWait = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
// Note: dag import doesn't log wait-flag-ignored like add does
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.False
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file with flag override
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=true")
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr
|
||||
// Flag should enable it despite config saying false
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
||||
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
||||
require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided
|
||||
})
|
||||
|
||||
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Import.FastProvideRoot = config.True
|
||||
})
|
||||
|
||||
node.StartDaemonWithReq(harness.RunRequest{
|
||||
CmdOpts: []harness.CmdOpt{
|
||||
harness.RunWithEnv(map[string]string{
|
||||
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
||||
}),
|
||||
},
|
||||
}, "")
|
||||
defer node.StopDaemon()
|
||||
|
||||
// Import CAR file with flag override
|
||||
r, err := os.Open(fixtureFile)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=false")
|
||||
require.NoError(t, err)
|
||||
|
||||
daemonLog := node.Daemon.Stderr.String()
|
||||
// Flag should disable it despite config saying true
|
||||
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
||||
})
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user