mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
fix(mfs): add soft limit for --flush=false (#10985)
* fix: add MFS operation limit for --flush=false adds a global counter that tracks consecutive MFS operations performed with --flush=false and fails with clear error after limit is reached. this prevents unbounded memory growth while avoiding the data corruption risks of auto-flushing. - adds Internal.MFSNoFlushLimit config - operations fail with actionable error at limit - counter resets on successful flush or any --flush=true operation - operations with --flush=true reset and don't count this commit removes automatic flush from https://github.com/ipfs/kubo/pull/10971 and instead errors to encourage users of --flush=false to develop a habit of calling 'ipfs files flush' periodically. boxo will no longer auto-flush (https://github.com/ipfs/boxo/pull/1041) to avoid corruption issues, and kubo applies the limit to 'ipfs files' commands instead. closes #10842 * test: add tests for MFSNoFlushLimit tests verify the new Internal.MFSNoFlushLimit config option: - default limit of 256 operations - custom limit configuration - counter reset on flush=true - counter reset on explicit flush command - limit=0 disables the feature - multiple MFS command types count towards limit * docs: explain why MFS operations fail instead of auto-flushing addresses feedback from https://github.com/ipfs/kubo/pull/10985#pullrequestreview-3256250970 - clarify that automatic flushing at limit was considered but rejected - explain the data corruption risks of auto-flushing - guide users who want auto-flush to use --flush=true (default) - document benefits of explicit failure for batch operations
This commit is contained in:
Marcin Rataj
GitHub
parent
f63887ae96
commit
a688b7eeac
@ -1,19 +1,23 @@
|
||||
package config
|
||||
|
||||
const (
|
||||
// DefaultMFSNoFlushLimit is the default limit for consecutive unflushed MFS operations
|
||||
DefaultMFSNoFlushLimit = 256
|
||||
)
|
||||
|
||||
type Internal struct {
|
||||
// All marked as omitempty since we are expecting to make changes to all subcomponents of Internal
|
||||
Bitswap *InternalBitswap `json:",omitempty"`
|
||||
UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold
|
||||
Libp2pForceReachability *OptionalString `json:",omitempty"`
|
||||
BackupBootstrapInterval *OptionalDuration `json:",omitempty"`
|
||||
// MFSAutoflushThreshold controls the number of entries cached in memory
|
||||
// for each MFS directory before auto-flush is triggered to prevent
|
||||
// unbounded memory growth when using --flush=false.
|
||||
// Default: 256 (matches HAMT shard size)
|
||||
// Set to 0 to disable cache limiting (old behavior, may cause high memory usage)
|
||||
// MFSNoFlushLimit controls the maximum number of consecutive
|
||||
// MFS operations allowed with --flush=false before requiring a manual flush.
|
||||
// This prevents unbounded memory growth and ensures data consistency.
|
||||
// Set to 0 to disable limiting (old behavior, may cause high memory usage)
|
||||
// This is an EXPERIMENTAL feature and may change or be removed in future releases.
|
||||
// See https://github.com/ipfs/kubo/issues/10842
|
||||
MFSAutoflushThreshold OptionalInteger `json:",omitempty"`
|
||||
MFSNoFlushLimit *OptionalInteger `json:",omitempty"`
|
||||
}
|
||||
|
||||
type InternalBitswap struct {
|
||||
|
||||
@ -11,6 +11,8 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
@ -35,6 +37,43 @@ import (
|
||||
|
||||
var flog = logging.Logger("cmds/files")
|
||||
|
||||
// Global counter for unflushed MFS operations
|
||||
var noFlushOperationCounter atomic.Int64
|
||||
|
||||
// Cached limit value (read once on first use)
|
||||
var (
|
||||
noFlushLimit int64
|
||||
noFlushLimitInit sync.Once
|
||||
)
|
||||
|
||||
// updateNoFlushCounter manages the counter for unflushed operations
|
||||
func updateNoFlushCounter(nd *core.IpfsNode, flush bool) error {
|
||||
if flush {
|
||||
// Reset counter when flushing
|
||||
noFlushOperationCounter.Store(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cache the limit on first use (config doesn't change at runtime)
|
||||
noFlushLimitInit.Do(func() {
|
||||
noFlushLimit = int64(config.DefaultMFSNoFlushLimit)
|
||||
if cfg, err := nd.Repo.Config(); err == nil && cfg.Internal.MFSNoFlushLimit != nil {
|
||||
noFlushLimit = cfg.Internal.MFSNoFlushLimit.WithDefault(int64(config.DefaultMFSNoFlushLimit))
|
||||
}
|
||||
})
|
||||
|
||||
// Check if limit reached
|
||||
if noFlushLimit > 0 && noFlushOperationCounter.Load() >= noFlushLimit {
|
||||
return fmt.Errorf("reached limit of %d unflushed MFS operations. "+
|
||||
"To resolve: 1) run 'ipfs files flush' to persist changes, "+
|
||||
"2) use --flush=true (default), or "+
|
||||
"3) increase Internal.MFSNoFlushLimit in config", noFlushLimit)
|
||||
}
|
||||
|
||||
noFlushOperationCounter.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
// FilesCmd is the 'ipfs files' command
|
||||
var FilesCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
@ -68,12 +107,14 @@ of consistency guarantees. If the daemon is unexpectedly killed before running
|
||||
'ipfs files flush' on the files in question, then data may be lost. This also
|
||||
applies to run 'ipfs repo gc' concurrently with '--flush=false' operations.
|
||||
|
||||
When using '--flush=false', directories will automatically flush when the
|
||||
number of cached entries exceeds the Internal.MFSAutoflushThreshold config.
|
||||
This prevents unbounded memory growth. We recommend flushing
|
||||
paths regularly with 'ipfs files flush', specially the folders on which many
|
||||
write operations are happening, as a way to clear the directory cache, free
|
||||
memory and speed up read operations.`,
|
||||
When using '--flush=false', operations are limited to prevent unbounded
|
||||
memory growth. After reaching Internal.MFSNoFlushLimit operations, further
|
||||
operations will fail until you run 'ipfs files flush'. This explicit failure
|
||||
(instead of auto-flushing) ensures you maintain control over when data is
|
||||
persisted, preventing unexpected partial states and making batch operations
|
||||
predictable. We recommend flushing paths regularly, especially folders with
|
||||
many write operations, to clear caches, free memory, and maintain good
|
||||
performance.`,
|
||||
},
|
||||
Options: []cmds.Option{
|
||||
cmds.BoolOption(filesFlushOptionName, "f", "Flush target and ancestors after write.").WithDefault(true),
|
||||
@ -516,12 +557,16 @@ being GC'ed.
|
||||
}
|
||||
}
|
||||
|
||||
flush, _ := req.Options[filesFlushOptionName].(bool)
|
||||
|
||||
if err := updateNoFlushCounter(nd, flush); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = mfs.PutNode(nd.FilesRoot, dst, node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cp: cannot put node in path %s: %s", dst, err)
|
||||
}
|
||||
|
||||
flush, _ := req.Options[filesFlushOptionName].(bool)
|
||||
if flush {
|
||||
if _, err := mfs.FlushPath(req.Context, nd.FilesRoot, dst); err != nil {
|
||||
return fmt.Errorf("cp: cannot flush the created file %s: %s", dst, err)
|
||||
@ -847,6 +892,10 @@ Example:
|
||||
|
||||
flush, _ := req.Options[filesFlushOptionName].(bool)
|
||||
|
||||
if err := updateNoFlushCounter(nd, flush); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
src, err := checkPath(req.Arguments[0])
|
||||
if err != nil {
|
||||
return err
|
||||
@ -984,6 +1033,10 @@ See '--to-files' in 'ipfs add --help' for more information.
|
||||
flush, _ := req.Options[filesFlushOptionName].(bool)
|
||||
rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)
|
||||
|
||||
if err := updateNoFlushCounter(nd, flush); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
|
||||
rawLeavesDef = true
|
||||
rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
|
||||
@ -1112,6 +1165,10 @@ Examples:
|
||||
|
||||
flush, _ := req.Options[filesFlushOptionName].(bool)
|
||||
|
||||
if err := updateNoFlushCounter(n, flush); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prefix, err := getPrefix(req)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -1164,6 +1221,9 @@ are run with the '--flush=false'.
|
||||
return err
|
||||
}
|
||||
|
||||
// Reset the counter (flush always resets)
|
||||
noFlushOperationCounter.Store(0)
|
||||
|
||||
return cmds.EmitOnce(res, &flushRes{enc.Encode(n.Cid())})
|
||||
},
|
||||
Type: flushRes{},
|
||||
|
||||
@ -246,13 +246,6 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Configure MFS directory cache auto-flush threshold if specified (experimental)
|
||||
cfg, err := repo.Config()
|
||||
if err == nil && !cfg.Internal.MFSAutoflushThreshold.IsDefault() {
|
||||
threshold := int(cfg.Internal.MFSAutoflushThreshold.WithDefault(int64(mfs.DefaultMaxCacheSize)))
|
||||
root.SetMaxCacheSize(threshold)
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
return root.Close()
|
||||
|
||||
@ -17,7 +17,8 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
|
||||
- [🎨 Updated WebUI](#-updated-webui)
|
||||
- [📌 Pin name improvements](#-pin-name-improvements)
|
||||
- [🛠️ Identity CID size enforcement and `ipfs files write` fixes](#️-identity-cid-size-enforcement-and-ipfs-files-write-fixes)
|
||||
- [Fix: Provide Filestore and Urlstore blocks on write](#️-provide-filestore-and-urlstore-blocks-on-write)
|
||||
- [📤 Provide Filestore and Urlstore blocks on write](#-provide-filestore-and-urlstore-blocks-on-write)
|
||||
- [🚦 MFS operation limit for --flush=false](#-mfs-operation-limit-for---flush=false)
|
||||
- [📦️ Important dependency updates](#-important-dependency-updates)
|
||||
- [📝 Changelog](#-changelog)
|
||||
- [👨👩👧👦 Contributors](#-contributors)
|
||||
@ -108,13 +109,13 @@ Identity CIDs use [multihash `0x00`](https://github.com/multiformats/multicodec/
|
||||
|
||||
This release resolves several long-standing MFS issues: raw nodes now preserve their codec instead of being forced to dag-pb, append operations on raw nodes work correctly by converting to UnixFS when needed, and identity CIDs properly inherit the full CID prefix from parent directories.
|
||||
|
||||
#### Provide Filestore and Urlstore blocks on write
|
||||
#### 📤 Provide Filestore and Urlstore blocks on write
|
||||
|
||||
Improvements to the providing system in the last release (provide blocks according to the configured Strategy) left out [Filestore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) and [Urlstore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore) blocks when the "all" strategy was used. They would only be reprovided but not provided on write. This is now fixed, and both Filestore blocks (local file references) and Urlstore blocks (HTTP/HTTPS URL references) will be provided correctly shortly after initial add.
|
||||
Improvements to the providing system in the last release (provide blocks according to the configured [Strategy](https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy)) left out [Filestore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) and [Urlstore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore) blocks when the "all" strategy was used. They would only be reprovided but not provided on write. This is now fixed, and both Filestore blocks (local file references) and Urlstore blocks (HTTP/HTTPS URL references) will be provided correctly shortly after initial add.
|
||||
|
||||
#### MFS directory cache auto-flush
|
||||
#### 🚦 MFS operation limit for --flush=false
|
||||
|
||||
The new [`Internal.MFSAutoflushThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalmfsautoflushthreshold) configuration option prevents unbounded memory growth when using `--flush=false` with `ipfs files` commands by automatically flushing directories when their cache exceeds the configured threshold (default: 256 entries).
|
||||
The new [`Internal.MFSNoFlushLimit`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalmfsnoflushlimit) configuration option prevents unbounded memory growth when using `--flush=false` with `ipfs files` commands. After performing the configured number of operations without flushing (default: 256), further operations will fail with a clear error message instructing users to flush manually.
|
||||
|
||||
### 📦️ Important dependency updates
|
||||
|
||||
|
||||
@ -1599,27 +1599,40 @@ Type: `flag`
|
||||
|
||||
**MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
|
||||
|
||||
### `Internal.MFSAutoflushThreshold`
|
||||
### `Internal.MFSNoFlushLimit`
|
||||
|
||||
Controls the number of entries cached in memory for each MFS directory before
|
||||
auto-flush is triggered to prevent unbounded memory growth when using `--flush=false`
|
||||
with `ipfs files` commands.
|
||||
Controls the maximum number of consecutive MFS operations allowed with `--flush=false`
|
||||
before requiring a manual flush. This prevents unbounded memory growth and ensures
|
||||
data consistency when using deferred flushing with `ipfs files` commands.
|
||||
|
||||
When a directory's cache reaches this threshold, it will automatically flush to
|
||||
the blockstore even when `--flush=false` is specified. This prevents excessive
|
||||
memory usage while still allowing performance benefits of deferred flushing for
|
||||
smaller operations.
|
||||
When the limit is reached, further operations will fail with an error message
|
||||
instructing the user to run `ipfs files flush`, use `--flush=true`, or increase
|
||||
this limit in the configuration.
|
||||
|
||||
**Examples:**
|
||||
* `256` - Default value. Provides a good balance between performance and memory usage.
|
||||
* `0` - Disables cache limiting (behavior before Kubo 0.38). May cause high memory
|
||||
usage with `--flush=false` on large directories.
|
||||
* `1024` - Higher limit for systems with more available memory that need to perform
|
||||
many operations before flushing.
|
||||
**Why operations fail instead of auto-flushing:** Automatic flushing once the limit
|
||||
is reached was considered but rejected because it can lead to data corruption issues
|
||||
that are difficult to debug. When the system decides to flush without user knowledge, it can:
|
||||
- Create partial states that violate user expectations about atomicity
|
||||
- Interfere with concurrent operations in unexpected ways
|
||||
- Make debugging and recovery much harder when issues occur
|
||||
|
||||
By failing explicitly, users maintain control over when their data is persisted,
|
||||
allowing them to:
|
||||
- Batch related operations together before flushing
|
||||
- Handle errors predictably at natural transaction boundaries
|
||||
- Understand exactly when and why their data is written to disk
|
||||
|
||||
If you expect automatic flushing behavior, simply use the default `--flush=true`
|
||||
(or omit the flag entirely) instead of `--flush=false`.
|
||||
|
||||
**⚠️ WARNING:** Increasing this limit or disabling it (setting to 0) can lead to:
|
||||
- **Out-of-memory errors (OOM)** - Each unflushed operation consumes memory
|
||||
- **Data loss** - If the daemon crashes before flushing, all unflushed changes are lost
|
||||
- **Degraded performance** - Large unflushed caches slow down MFS operations
|
||||
|
||||
Default: `256`
|
||||
|
||||
Type: `optionalInteger` (0 disables the limit, risky, may lead to errors)
|
||||
Type: `optionalInteger` (0 disables the limit, strongly discouraged)
|
||||
|
||||
**Note:** This is an EXPERIMENTAL feature and may change or be removed in future releases.
|
||||
See [#10842](https://github.com/ipfs/kubo/issues/10842) for more information.
|
||||
|
||||
@ -7,7 +7,7 @@ go 1.25
|
||||
replace github.com/ipfs/kubo => ./../../..
|
||||
|
||||
require (
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28
|
||||
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
|
||||
github.com/libp2p/go-libp2p v0.43.0
|
||||
github.com/multiformats/go-multiaddr v0.16.1
|
||||
|
||||
@ -289,8 +289,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28 h1:kDoj2V7ghhLdQeQUtzr605tb6NJ4AzwRYtXFJas+Wyc=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
|
||||
|
||||
2
go.mod
2
go.mod
@ -22,7 +22,7 @@ require (
|
||||
github.com/hashicorp/go-version v1.7.0
|
||||
github.com/ipfs-shipyard/nopfs v0.0.14
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28
|
||||
github.com/ipfs/go-block-format v0.2.3
|
||||
github.com/ipfs/go-cid v0.5.0
|
||||
github.com/ipfs/go-cidutil v0.1.0
|
||||
|
||||
4
go.sum
4
go.sum
@ -356,8 +356,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28 h1:kDoj2V7ghhLdQeQUtzr605tb6NJ4AzwRYtXFJas+Wyc=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -178,3 +179,163 @@ func TestFilesRm(t *testing.T) {
|
||||
assert.NotContains(t, lsRes.Stdout.String(), "test-dir")
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilesNoFlushLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("reaches default limit of 256 operations", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
|
||||
// Perform 256 operations with --flush=false (should succeed)
|
||||
for i := 0; i < 256; i++ {
|
||||
res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i))
|
||||
assert.NoError(t, res.Err, "operation %d should succeed", i+1)
|
||||
}
|
||||
|
||||
// 257th operation should fail
|
||||
res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir256")
|
||||
require.NotNil(t, res.ExitErr, "command should have failed")
|
||||
assert.NotEqual(t, 0, res.ExitErr.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "reached limit of 256 unflushed MFS operations")
|
||||
assert.Contains(t, res.Stderr.String(), "run 'ipfs files flush'")
|
||||
assert.Contains(t, res.Stderr.String(), "use --flush=true")
|
||||
assert.Contains(t, res.Stderr.String(), "increase Internal.MFSNoFlushLimit")
|
||||
})
|
||||
|
||||
t.Run("custom limit via config", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set custom limit to 5
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
limit := config.NewOptionalInteger(5)
|
||||
cfg.Internal.MFSNoFlushLimit = limit
|
||||
})
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
// Perform 5 operations (should succeed)
|
||||
for i := 0; i < 5; i++ {
|
||||
res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i))
|
||||
assert.NoError(t, res.Err, "operation %d should succeed", i+1)
|
||||
}
|
||||
|
||||
// 6th operation should fail
|
||||
res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir5")
|
||||
require.NotNil(t, res.ExitErr, "command should have failed")
|
||||
assert.NotEqual(t, 0, res.ExitErr.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "reached limit of 5 unflushed MFS operations")
|
||||
})
|
||||
|
||||
t.Run("flush=true resets counter", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set limit to 3 for faster testing
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
limit := config.NewOptionalInteger(3)
|
||||
cfg.Internal.MFSNoFlushLimit = limit
|
||||
})
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
// Do 2 operations with --flush=false
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/dir1")
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/dir2")
|
||||
|
||||
// Operation with --flush=true should reset counter
|
||||
node.IPFS("files", "mkdir", "--flush=true", "/dir3")
|
||||
|
||||
// Now we should be able to do 3 more operations with --flush=false
|
||||
for i := 4; i <= 6; i++ {
|
||||
res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i))
|
||||
assert.NoError(t, res.Err, "operation after flush should succeed")
|
||||
}
|
||||
|
||||
// 4th operation after reset should fail
|
||||
res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir7")
|
||||
require.NotNil(t, res.ExitErr, "command should have failed")
|
||||
assert.NotEqual(t, 0, res.ExitErr.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "reached limit of 3 unflushed MFS operations")
|
||||
})
|
||||
|
||||
t.Run("explicit flush command resets counter", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set limit to 3 for faster testing
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
limit := config.NewOptionalInteger(3)
|
||||
cfg.Internal.MFSNoFlushLimit = limit
|
||||
})
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
// Do 2 operations with --flush=false
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/dir1")
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/dir2")
|
||||
|
||||
// Explicit flush should reset counter
|
||||
node.IPFS("files", "flush")
|
||||
|
||||
// Now we should be able to do 3 more operations
|
||||
for i := 3; i <= 5; i++ {
|
||||
res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i))
|
||||
assert.NoError(t, res.Err, "operation after flush should succeed")
|
||||
}
|
||||
|
||||
// 4th operation should fail
|
||||
res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir6")
|
||||
require.NotNil(t, res.ExitErr, "command should have failed")
|
||||
assert.NotEqual(t, 0, res.ExitErr.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "reached limit of 3 unflushed MFS operations")
|
||||
})
|
||||
|
||||
t.Run("limit=0 disables the feature", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set limit to 0 (disabled)
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
limit := config.NewOptionalInteger(0)
|
||||
cfg.Internal.MFSNoFlushLimit = limit
|
||||
})
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
// Should be able to do many operations without error
|
||||
for i := 0; i < 300; i++ {
|
||||
res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i))
|
||||
assert.NoError(t, res.Err, "operation %d should succeed with limit disabled", i+1)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different MFS commands count towards limit", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
// Set limit to 5 for testing
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
limit := config.NewOptionalInteger(5)
|
||||
cfg.Internal.MFSNoFlushLimit = limit
|
||||
})
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
// Mix of different MFS operations (5 operations to hit the limit)
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/testdir")
|
||||
// Create a file first, then copy it
|
||||
testCid := node.IPFSAddStr("test content")
|
||||
node.IPFS("files", "cp", "--flush=false", fmt.Sprintf("/ipfs/%s", testCid), "/testfile")
|
||||
node.IPFS("files", "cp", "--flush=false", "/testfile", "/testfile2")
|
||||
node.IPFS("files", "mv", "--flush=false", "/testfile2", "/testfile3")
|
||||
node.IPFS("files", "mkdir", "--flush=false", "/anotherdir")
|
||||
|
||||
// 6th operation should fail
|
||||
res := node.RunIPFS("files", "mkdir", "--flush=false", "/another")
|
||||
require.NotNil(t, res.ExitErr, "command should have failed")
|
||||
assert.NotEqual(t, 0, res.ExitErr.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "reached limit of 5 unflushed MFS operations")
|
||||
})
|
||||
}
|
||||
|
||||
@ -135,7 +135,7 @@ require (
|
||||
github.com/huin/goupnp v1.3.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/ipfs/bbloom v0.0.4 // indirect
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 // indirect
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28 // indirect
|
||||
github.com/ipfs/go-bitfield v1.1.0 // indirect
|
||||
github.com/ipfs/go-block-format v0.2.3 // indirect
|
||||
github.com/ipfs/go-cid v0.5.0 // indirect
|
||||
|
||||
@ -332,8 +332,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28 h1:kDoj2V7ghhLdQeQUtzr605tb6NJ4AzwRYtXFJas+Wyc=
|
||||
github.com/ipfs/boxo v0.34.1-0.20250925224331-260f4b387f28/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user