From f5427b556a20417e52f8998d95cff613e6f789b7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 17 Jan 2026 03:11:14 +0100 Subject: [PATCH 01/22] feat(config): Import.* and unixfs-v1-2025 profile implements IPIP-499: add config options for controlling UnixFS DAG determinism and introduces `unixfs-v1-2025` and `unixfs-v0-2015` profiles for cross-implementation CID reproducibility. changes: - add Import.* fields: HAMTDirectorySizeEstimation, SymlinkMode, DAGLayout, IncludeEmptyDirectories, IncludeHidden - add validation for all Import.* config values - add unixfs-v1-2025 profile (recommended for new data) - add unixfs-v0-2015 profile (alias: legacy-cid-v0) - remove deprecated test-cid-v1 and test-cid-v1-wide profiles - wire Import.HAMTSizeEstimationMode() to boxo globals - update go.mod to use boxo with SizeEstimationMode support ref: https://specs.ipfs.tech/ipips/ipip-0499/ --- config/import.go | 99 +++++++++++++++--- config/import_test.go | 139 +++++++++++++++++++++++++ config/profile.go | 69 ++++++------ core/node/groups.go | 5 +- docs/config.md | 109 +++++++++++++------ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 11 files changed, 350 insertions(+), 89 deletions(-) diff --git a/config/import.go b/config/import.go index d595199c8..56511a079 100644 --- a/config/import.go +++ b/config/import.go @@ -29,6 +29,23 @@ const ( // write-batch. The total size of the batch is limited by // BatchMaxnodes and BatchMaxSize. DefaultBatchMaxSize = 100 << 20 // 20MiB + + // HAMTSizeEstimation values for Import.UnixFSHAMTDirectorySizeEstimation + HAMTSizeEstimationLinks = "links" // legacy: estimate using link names + CID byte lengths + HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size + HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely + + // SymlinkMode values for Import.UnixFSSymlinkMode + SymlinkModePreserve = "preserve" // preserve symlinks as UnixFS symlink nodes + SymlinkModeDereference = "dereference" // dereference symlinks, import target content + + // DAGLayout values for Import.UnixFSDAGLayout + DAGLayoutBalanced = "balanced" // balanced DAG layout (default) + DAGLayoutTrickle = "trickle" // trickle DAG layout + + DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior + DefaultUnixFSSymlinkMode = SymlinkModePreserve // preserve symlinks as UnixFS symlink nodes + DefaultUnixFSDAGLayout = DAGLayoutBalanced // balanced DAG layout ) var ( @@ -40,18 +57,21 @@ var ( // Import configures the default options for ingesting data. This affects commands // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'. type Import struct { - CidVersion OptionalInteger - UnixFSRawLeaves Flag - UnixFSChunker OptionalString - HashFunction OptionalString - UnixFSFileMaxLinks OptionalInteger - UnixFSDirectoryMaxLinks OptionalInteger - UnixFSHAMTDirectoryMaxFanout OptionalInteger - UnixFSHAMTDirectorySizeThreshold OptionalBytes - BatchMaxNodes OptionalInteger - BatchMaxSize OptionalInteger - FastProvideRoot Flag - FastProvideWait Flag + CidVersion OptionalInteger + UnixFSRawLeaves Flag + UnixFSChunker OptionalString + HashFunction OptionalString + UnixFSFileMaxLinks OptionalInteger + UnixFSDirectoryMaxLinks OptionalInteger + UnixFSHAMTDirectoryMaxFanout OptionalInteger + UnixFSHAMTDirectorySizeThreshold OptionalBytes + UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled" + UnixFSSymlinkMode OptionalString // "preserve" or "dereference" + UnixFSDAGLayout OptionalString // "balanced" or "trickle" + BatchMaxNodes OptionalInteger + BatchMaxSize OptionalInteger + FastProvideRoot Flag + FastProvideWait Flag } // ValidateImportConfig validates the Import configuration according to UnixFS spec requirements. @@ -129,6 +149,42 @@ func ValidateImportConfig(cfg *Import) error { } } + // Validate UnixFSHAMTDirectorySizeEstimation + if !cfg.UnixFSHAMTDirectorySizeEstimation.IsDefault() { + est := cfg.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) + switch est { + case HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled: + // valid + default: + return fmt.Errorf("Import.UnixFSHAMTDirectorySizeEstimation must be %q, %q, or %q, got %q", + HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled, est) + } + } + + // Validate UnixFSSymlinkMode + if !cfg.UnixFSSymlinkMode.IsDefault() { + mode := cfg.UnixFSSymlinkMode.WithDefault(DefaultUnixFSSymlinkMode) + switch mode { + case SymlinkModePreserve, SymlinkModeDereference: + // valid + default: + return fmt.Errorf("Import.UnixFSSymlinkMode must be %q or %q, got %q", + SymlinkModePreserve, SymlinkModeDereference, mode) + } + } + + // Validate UnixFSDAGLayout + if !cfg.UnixFSDAGLayout.IsDefault() { + layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout) + switch layout { + case DAGLayoutBalanced, DAGLayoutTrickle: + // valid + default: + return fmt.Errorf("Import.UnixFSDAGLayout must be %q or %q, got %q", + DAGLayoutBalanced, DAGLayoutTrickle, layout) + } + } + return nil } @@ -144,8 +200,7 @@ func isValidChunker(chunker string) bool { } // Check for size- format - if strings.HasPrefix(chunker, "size-") { - sizeStr := strings.TrimPrefix(chunker, "size-") + if sizeStr, ok := strings.CutPrefix(chunker, "size-"); ok { if sizeStr == "" { return false } @@ -167,7 +222,7 @@ func isValidChunker(chunker string) bool { // Parse and validate min, avg, max values values := make([]int, 3) - for i := 0; i < 3; i++ { + for i := range 3 { val, err := strconv.Atoi(parts[i+1]) if err != nil { return false @@ -182,3 +237,17 @@ func isValidChunker(chunker string) bool { return false } + +// HAMTSizeEstimationMode returns the boxo SizeEstimationMode based on the config value. +func (i *Import) HAMTSizeEstimationMode() io.SizeEstimationMode { + switch i.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) { + case HAMTSizeEstimationLinks: + return io.SizeEstimationLinks + case HAMTSizeEstimationBlock: + return io.SizeEstimationBlock + case HAMTSizeEstimationDisabled: + return io.SizeEstimationDisabled + default: + return io.SizeEstimationLinks + } +} diff --git a/config/import_test.go b/config/import_test.go index f045b9751..31e6dcf56 100644 --- a/config/import_test.go +++ b/config/import_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/ipfs/boxo/ipld/unixfs/io" mh "github.com/multiformats/go-multihash" ) @@ -406,3 +407,141 @@ func TestIsPowerOfTwo(t *testing.T) { }) } } + +func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) { + tests := []struct { + name string + value string + wantErr bool + errMsg string + }{ + {name: "valid links", value: HAMTSizeEstimationLinks, wantErr: false}, + {name: "valid block", value: HAMTSizeEstimationBlock, wantErr: false}, + {name: "valid disabled", value: HAMTSizeEstimationDisabled, wantErr: false}, + {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, + {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, + {name: "invalid typo", value: "link", wantErr: true, errMsg: "must be"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Import{ + UnixFSHAMTDirectorySizeEstimation: *NewOptionalString(tt.value), + } + + err := ValidateImportConfig(cfg) + + if tt.wantErr { + if err == nil { + t.Errorf("expected error for value=%q, got nil", tt.value) + } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("error = %v, want error containing %q", err, tt.errMsg) + } + } else { + if err != nil { + t.Errorf("unexpected error for value=%q: %v", tt.value, err) + } + } + }) + } +} + +func TestValidateImportConfig_SymlinkMode(t *testing.T) { + tests := []struct { + name string + value string + wantErr bool + errMsg string + }{ + {name: "valid preserve", value: SymlinkModePreserve, wantErr: false}, + {name: "valid dereference", value: SymlinkModeDereference, wantErr: false}, + {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, + {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, + {name: "invalid follow", value: "follow", wantErr: true, errMsg: "must be"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Import{ + UnixFSSymlinkMode: *NewOptionalString(tt.value), + } + + err := ValidateImportConfig(cfg) + + if tt.wantErr { + if err == nil { + t.Errorf("expected error for value=%q, got nil", tt.value) + } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("error = %v, want error containing %q", err, tt.errMsg) + } + } else { + if err != nil { + t.Errorf("unexpected error for value=%q: %v", tt.value, err) + } + } + }) + } +} + +func TestValidateImportConfig_DAGLayout(t *testing.T) { + tests := []struct { + name string + value string + wantErr bool + errMsg string + }{ + {name: "valid balanced", value: DAGLayoutBalanced, wantErr: false}, + {name: "valid trickle", value: DAGLayoutTrickle, wantErr: false}, + {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, + {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, + {name: "invalid flat", value: "flat", wantErr: true, errMsg: "must be"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Import{ + UnixFSDAGLayout: *NewOptionalString(tt.value), + } + + err := ValidateImportConfig(cfg) + + if tt.wantErr { + if err == nil { + t.Errorf("expected error for value=%q, got nil", tt.value) + } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("error = %v, want error containing %q", err, tt.errMsg) + } + } else { + if err != nil { + t.Errorf("unexpected error for value=%q: %v", tt.value, err) + } + } + }) + } +} + +func TestImport_HAMTSizeEstimationMode(t *testing.T) { + tests := []struct { + cfg string + want io.SizeEstimationMode + }{ + {HAMTSizeEstimationLinks, io.SizeEstimationLinks}, + {HAMTSizeEstimationBlock, io.SizeEstimationBlock}, + {HAMTSizeEstimationDisabled, io.SizeEstimationDisabled}, + {"", io.SizeEstimationLinks}, // default (unset returns default) + {"unknown", io.SizeEstimationLinks}, // fallback to default + } + + for _, tt := range tests { + t.Run(tt.cfg, func(t *testing.T) { + var imp Import + if tt.cfg != "" { + imp.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(tt.cfg) + } + got := imp.HAMTSizeEstimationMode() + if got != tt.want { + t.Errorf("Import.HAMTSizeEstimationMode() with %q = %v, want %v", tt.cfg, got, tt.want) + } + }) + } +} diff --git a/config/profile.go b/config/profile.go index 692688796..904a32497 100644 --- a/config/profile.go +++ b/config/profile.go @@ -312,45 +312,34 @@ fetching may be degraded. return nil }, }, + "unixfs-v0-2015": { + Description: `Legacy UnixFS import profile for backward-compatible CID generation. +Produces CIDv0 with no raw leaves, sha2-256, 256 KiB chunks, and +link-based HAMT size estimation. Use only when legacy CIDs are required. +See https://github.com/ipfs/specs/pull/499. Alias: legacy-cid-v0`, + Transform: applyUnixFSv02015, + }, "legacy-cid-v0": { - Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. This is likely the least optimal preset, use only if legacy behavior is required.`, - Transform: func(c *Config) error { - c.Import.CidVersion = *NewOptionalInteger(0) - c.Import.UnixFSRawLeaves = False - c.Import.UnixFSChunker = *NewOptionalString("size-262144") - c.Import.HashFunction = *NewOptionalString("sha2-256") - c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174) - c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) - c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) - c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") - return nil - }, + Description: `Alias for unixfs-v0-2015 profile.`, + Transform: applyUnixFSv02015, }, - "test-cid-v1": { - Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1 MiB chunks (max 174 links per file, 256 per HAMT node, switch dir to HAMT above 256KiB).`, + "unixfs-v1-2025": { + Description: `Recommended UnixFS import profile for cross-implementation CID determinism. +Uses CIDv1, raw leaves, sha2-256, 1 MiB chunks, 1024 links per file node, +256 HAMT fanout, and block-based size estimation for HAMT threshold. +See https://github.com/ipfs/specs/pull/499`, Transform: func(c *Config) error { c.Import.CidVersion = *NewOptionalInteger(1) c.Import.UnixFSRawLeaves = True - c.Import.UnixFSChunker = *NewOptionalString("size-1048576") - c.Import.HashFunction = *NewOptionalString("sha2-256") - c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174) - c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) - c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) - c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") - return nil - }, - }, - "test-cid-v1-wide": { - Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1MiB chunks and wider file DAGs (max 1024 links per every node type, switch dir to HAMT above 1MiB).`, - Transform: func(c *Config) error { - c.Import.CidVersion = *NewOptionalInteger(1) - c.Import.UnixFSRawLeaves = True - c.Import.UnixFSChunker = *NewOptionalString("size-1048576") // 1MiB + c.Import.UnixFSChunker = *NewOptionalString("size-1048576") // 1 MiB c.Import.HashFunction = *NewOptionalString("sha2-256") c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(1024) - c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) // no limit here, use size-based Import.UnixFSHAMTDirectorySizeThreshold instead - c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(1024) - c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("1MiB") // 1MiB + c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) + c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) + c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") + c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock) + c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) + c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil }, }, @@ -435,3 +424,19 @@ func mapKeys(m map[string]struct{}) []string { } return out } + +// applyUnixFSv02015 applies the legacy UnixFS v0 (2015) import settings. +func applyUnixFSv02015(c *Config) error { + c.Import.CidVersion = *NewOptionalInteger(0) + c.Import.UnixFSRawLeaves = False + c.Import.UnixFSChunker = *NewOptionalString("size-262144") // 256 KiB + c.Import.HashFunction = *NewOptionalString("sha2-256") + c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174) + c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) + c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) + c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") + c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks) + c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) + c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) + return nil +} diff --git a/core/node/groups.go b/core/node/groups.go index bacc12160..ba6348db8 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -439,11 +439,12 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { } // Auto-sharding settings - shardSingThresholdInt := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) + shardSizeThreshold := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) shardMaxFanout := cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout) // TODO: avoid overriding this globally, see if we can extend Directory interface like Get/SetMaxLinks from https://github.com/ipfs/boxo/pull/906 - uio.HAMTShardingSize = int(shardSingThresholdInt) + uio.HAMTShardingSize = int(shardSizeThreshold) uio.DefaultShardWidth = int(shardMaxFanout) + uio.HAMTSizeEstimation = cfg.Import.HAMTSizeEstimationMode() providerStrategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) diff --git a/docs/config.md b/docs/config.md index e6ab44d04..16987b0ce 100644 --- a/docs/config.md +++ b/docs/config.md @@ -242,6 +242,9 @@ config file at runtime. - [`Import.UnixFSDirectoryMaxLinks`](#importunixfsdirectorymaxlinks) - [`Import.UnixFSHAMTDirectoryMaxFanout`](#importunixfshamtdirectorymaxfanout) - [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) + - [`Import.UnixFSHAMTDirectorySizeEstimation`](#importunixfshamtdirectorysizeestimation) + - [`Import.UnixFSSymlinkMode`](#importunixfssymlinkmode) + - [`Import.UnixFSDAGLayout`](#importunixfsdaglayout) - [`Version`](#version) - [`Version.AgentSuffix`](#versionagentsuffix) - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled) @@ -263,9 +266,9 @@ config file at runtime. - [`lowpower` profile](#lowpower-profile) - [`announce-off` profile](#announce-off-profile) - [`announce-on` profile](#announce-on-profile) + - [`unixfs-v0-2015` profile](#unixfs-v0-2015-profile) - [`legacy-cid-v0` profile](#legacy-cid-v0-profile) - - [`test-cid-v1` profile](#test-cid-v1-profile) - - [`test-cid-v1-wide` profile](#test-cid-v1-wide-profile) + - [`unixfs-v1-2025` profile](#unixfs-v1-2025-profile) - [Security](#security) - [Port and Network Exposure](#port-and-network-exposure) - [Security Best Practices](#security-best-practices) @@ -3821,6 +3824,57 @@ Default: `256KiB` (may change, inspect `DefaultUnixFSHAMTDirectorySizeThreshold` Type: [`optionalBytes`](#optionalbytes) +### `Import.UnixFSHAMTDirectorySizeEstimation` + +Controls how directory size is estimated when deciding whether to switch +from a basic UnixFS directory to HAMT sharding. + +Accepted values: + +- `links` (default): Legacy estimation using sum of link names and CID byte lengths. +- `block`: Full serialized dag-pb block size for accurate threshold decisions. +- `disabled`: Disable HAMT sharding entirely (directories always remain basic). + +The `block` estimation is recommended for new profiles as it provides more +accurate threshold decisions and better cross-implementation consistency. +See [IPIP-499](https://github.com/ipfs/specs/pull/499) for more details. + +Commands affected: `ipfs add` + +Default: `links` + +Type: `optionalString` + +### `Import.UnixFSSymlinkMode` + +Controls how symbolic links are handled during import. + +Accepted values: + +- `preserve` (default): Store symlinks as UnixFS symlink nodes containing the target path. +- `dereference`: Follow symlinks and import the target content instead. + +Commands affected: `ipfs add` + +Default: `preserve` + +Type: `optionalString` + +### `Import.UnixFSDAGLayout` + +Controls the DAG layout used when chunking files. + +Accepted values: + +- `balanced` (default): Balanced DAG layout with uniform leaf depth. +- `trickle`: Trickle DAG layout optimized for streaming. + +Commands affected: `ipfs add` + +Default: `balanced` + +Type: `optionalString` + ## `Version` Options to configure agent version announced to the swarm, and leveraging @@ -4021,42 +4075,35 @@ Disables [Provide](#provide) system (and announcing to Amino DHT). (Re-)enables [Provide](#provide) system (reverts [`announce-off` profile](#announce-off-profile)). +### `unixfs-v0-2015` profile + +Legacy UnixFS import profile for backward-compatible CID generation. +Produces CIDv0 with no raw leaves, sha2-256, 256 KiB chunks, and +link-based HAMT size estimation. + +See for exact [`Import.*`](#import) settings. + +> [!NOTE] +> Use only when legacy CIDs are required. For new projects, use [`unixfs-v1-2025`](#unixfs-v1-2025-profile). +> +> See [IPIP-499](https://github.com/ipfs/specs/pull/499) for more details. + ### `legacy-cid-v0` profile -Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. +Alias for [`unixfs-v0-2015`](#unixfs-v0-2015-profile) profile. + +### `unixfs-v1-2025` profile + +Recommended UnixFS import profile for cross-implementation CID determinism. +Uses CIDv1, raw leaves, sha2-256, 1 MiB chunks, 1024 links per file node, +256 HAMT fanout, and block-based size estimation for HAMT threshold. See for exact [`Import.*`](#import) settings. > [!NOTE] -> This profile is provided for legacy users and should not be used for new projects. - -### `test-cid-v1` profile - -Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 -and 1 MiB chunks (max 174 links per file, 256 per HAMT node, switch dir to HAMT -above 256KiB). - -See for exact [`Import.*`](#import) settings. - -> [!NOTE] -> [`Import.*`](#import) settings applied by this profile MAY change in future release. Provided for testing purposes. +> This profile ensures CID consistency across different IPFS implementations. > -> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details, -> and provide feedback in [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499). - -### `test-cid-v1-wide` profile - -Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 -and 1 MiB chunks and wider file DAGs (max 1024 links per every node type, -switch dir to HAMT above 1MiB). - -See for exact [`Import.*`](#import) settings. - -> [!NOTE] -> [`Import.*`](#import) settings applied by this profile MAY change in future release. Provided for testing purposes. -> -> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details, -> and provide feedback in [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499). +> See [IPIP-499](https://github.com/ipfs/specs/pull/499) for more details. ## Security diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 698eccfa3..234df7d5d 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 + github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index dba9d67aa..f998634d5 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 h1:pRQYSSGnGQa921d8v0uhXg2BGzoSf9ndTWTlR7ImVoo= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 109a2ca99..37ceedbf7 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 + github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index afed01f14..4e1c346a1 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 h1:pRQYSSGnGQa921d8v0uhXg2BGzoSf9ndTWTlR7ImVoo= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 98bf7ebf1..234086a25 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 // indirect + github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 32d404247..c99ca0aa4 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2 h1:pRQYSSGnGQa921d8v0uhXg2BGzoSf9ndTWTlR7ImVoo= -github.com/ipfs/boxo v0.35.3-0.20260109213916-89dc184784f2/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= +github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 01b1ce0cca2cbd3f2dd850d054c5820c8e1ca0f8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 17 Jan 2026 05:51:28 +0100 Subject: [PATCH 02/22] feat(add): add --dereference-symlinks, --empty-dirs, --hidden CLI flags add CLI flags for controlling file collection behavior during ipfs add: - `--dereference-symlinks`: recursively resolve symlinks to their target content (replaces deprecated --dereference-args which only worked on CLI arguments). wired through go-ipfs-cmds to boxo's SerialFileOptions. - `--empty-dirs` / `-E`: include empty directories (default: true) - `--hidden` / `-H`: include hidden files (default: false) these flags are CLI-only and not wired to Import.* config options because go-ipfs-cmds library handles input file filtering before the directory tree is passed to kubo. removed unused Import.UnixFSSymlinkMode config option that was defined but never actually read by the CLI. also: - wire --trickle to Import.UnixFSDAGLayout config default - update go-ipfs-cmds to v0.15.1-0.20260117043932-17687e216294 - add SYMLINK HANDLING section to ipfs add help text - add CLI tests for all three flags ref: https://github.com/ipfs/specs/pull/499 --- config/import.go | 19 +-- config/import_test.go | 39 +----- config/profile.go | 2 - core/commands/add.go | 34 ++++- core/coreapi/unixfs.go | 3 + core/coreiface/options/unixfs.go | 29 ++-- core/coreunix/add.go | 60 ++++++--- docs/changelogs/v0.40.md | 38 ++++++ docs/config.md | 24 +--- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/cli/add_test.go | 180 +++++++++++++++++++++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 16 files changed, 331 insertions(+), 115 deletions(-) diff --git a/config/import.go b/config/import.go index 56511a079..995c371d2 100644 --- a/config/import.go +++ b/config/import.go @@ -35,17 +35,13 @@ const ( HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely - // SymlinkMode values for Import.UnixFSSymlinkMode - SymlinkModePreserve = "preserve" // preserve symlinks as UnixFS symlink nodes - SymlinkModeDereference = "dereference" // dereference symlinks, import target content - // DAGLayout values for Import.UnixFSDAGLayout DAGLayoutBalanced = "balanced" // balanced DAG layout (default) DAGLayoutTrickle = "trickle" // trickle DAG layout DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior - DefaultUnixFSSymlinkMode = SymlinkModePreserve // preserve symlinks as UnixFS symlink nodes DefaultUnixFSDAGLayout = DAGLayoutBalanced // balanced DAG layout + DefaultUnixFSIncludeEmptyDirs = true // include empty directories ) var ( @@ -66,7 +62,6 @@ type Import struct { UnixFSHAMTDirectoryMaxFanout OptionalInteger UnixFSHAMTDirectorySizeThreshold OptionalBytes UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled" - UnixFSSymlinkMode OptionalString // "preserve" or "dereference" UnixFSDAGLayout OptionalString // "balanced" or "trickle" BatchMaxNodes OptionalInteger BatchMaxSize OptionalInteger @@ -161,18 +156,6 @@ func ValidateImportConfig(cfg *Import) error { } } - // Validate UnixFSSymlinkMode - if !cfg.UnixFSSymlinkMode.IsDefault() { - mode := cfg.UnixFSSymlinkMode.WithDefault(DefaultUnixFSSymlinkMode) - switch mode { - case SymlinkModePreserve, SymlinkModeDereference: - // valid - default: - return fmt.Errorf("Import.UnixFSSymlinkMode must be %q or %q, got %q", - SymlinkModePreserve, SymlinkModeDereference, mode) - } - } - // Validate UnixFSDAGLayout if !cfg.UnixFSDAGLayout.IsDefault() { layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout) diff --git a/config/import_test.go b/config/import_test.go index 31e6dcf56..5d9605c1d 100644 --- a/config/import_test.go +++ b/config/import_test.go @@ -446,43 +446,6 @@ func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) { } } -func TestValidateImportConfig_SymlinkMode(t *testing.T) { - tests := []struct { - name string - value string - wantErr bool - errMsg string - }{ - {name: "valid preserve", value: SymlinkModePreserve, wantErr: false}, - {name: "valid dereference", value: SymlinkModeDereference, wantErr: false}, - {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, - {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, - {name: "invalid follow", value: "follow", wantErr: true, errMsg: "must be"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := &Import{ - UnixFSSymlinkMode: *NewOptionalString(tt.value), - } - - err := ValidateImportConfig(cfg) - - if tt.wantErr { - if err == nil { - t.Errorf("expected error for value=%q, got nil", tt.value) - } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { - t.Errorf("error = %v, want error containing %q", err, tt.errMsg) - } - } else { - if err != nil { - t.Errorf("unexpected error for value=%q: %v", tt.value, err) - } - } - }) - } -} - func TestValidateImportConfig_DAGLayout(t *testing.T) { tests := []struct { name string @@ -528,7 +491,7 @@ func TestImport_HAMTSizeEstimationMode(t *testing.T) { {HAMTSizeEstimationLinks, io.SizeEstimationLinks}, {HAMTSizeEstimationBlock, io.SizeEstimationBlock}, {HAMTSizeEstimationDisabled, io.SizeEstimationDisabled}, - {"", io.SizeEstimationLinks}, // default (unset returns default) + {"", io.SizeEstimationLinks}, // default (unset returns default) {"unknown", io.SizeEstimationLinks}, // fallback to default } diff --git a/config/profile.go b/config/profile.go index 904a32497..7832c7599 100644 --- a/config/profile.go +++ b/config/profile.go @@ -338,7 +338,6 @@ See https://github.com/ipfs/specs/pull/499`, c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock) - c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil }, @@ -436,7 +435,6 @@ func applyUnixFSv02015(c *Config) error { c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks) - c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil } diff --git a/core/commands/add.go b/core/commands/add.go index cb4bcb312..a56b69434 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -68,6 +68,7 @@ const ( mtimeNsecsOptionName = "mtime-nsecs" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" + emptyDirsOptionName = "empty-dirs" ) const ( @@ -147,6 +148,18 @@ to find it in the future: See 'ipfs files --help' to learn more about using MFS for keeping track of added files and directories. +SYMLINK HANDLING: + +By default, symbolic links are preserved as UnixFS symlink nodes that store +the target path. Use --dereference-symlinks to resolve symlinks to their +target content instead: + + > ipfs add -r --dereference-symlinks ./mydir + +This recursively resolves all symlinks encountered during directory traversal. +Symlinks to files become regular file content, symlinks to directories are +traversed and their contents are added. + CHUNKING EXAMPLES: The chunker option, '-s', specifies the chunking strategy that dictates @@ -200,11 +213,13 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import Options: []cmds.Option{ // Input Processing cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) - cmds.OptionDerefArgs, // a builtin option that resolves passed in filesystem links (--dereference-args) + cmds.OptionDerefArgs, // DEPRECATED: use --dereference-symlinks instead cmds.OptionStdinName, // a builtin option that optionally allows wrapping stdin into a named file cmds.OptionHidden, cmds.OptionIgnore, cmds.OptionIgnoreRules, + cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs), + cmds.OptionDerefSymlinks, // resolve symlinks to their target content // Output Control cmds.BoolOption(quietOptionName, "q", "Write minimal output."), cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."), @@ -274,7 +289,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import } progress, _ := req.Options[progressOptionName].(bool) - trickle, _ := req.Options[trickleOptionName].(bool) + trickle, trickleSet := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) onlyHash, _ := req.Options[onlyHashOptionName].(bool) silent, _ := req.Options[silentOptionName].(bool) @@ -312,6 +327,19 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) + emptyDirs, _ := req.Options[emptyDirsOptionName].(bool) + + // Handle --dereference-args deprecation + derefArgs, derefArgsSet := req.Options[cmds.DerefLong].(bool) + if derefArgsSet && derefArgs { + return fmt.Errorf("--dereference-args is deprecated: use --dereference-symlinks instead") + } + + // Wire --trickle from config + if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() { + layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout) + trickle = layout == config.DAGLayoutTrickle + } if chunker == "" { chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) @@ -409,6 +437,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import options.Unixfs.PreserveMode(preserveMode), options.Unixfs.PreserveMtime(preserveMtime), + + options.Unixfs.IncludeEmptyDirs(emptyDirs), } if mode != 0 { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 7f068a227..a5507f557 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -183,6 +183,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.PreserveMtime = settings.PreserveMtime fileAdder.FileMode = settings.Mode fileAdder.FileMtime = settings.Mtime + if settings.IncludeEmptyDirsSet { + fileAdder.IncludeEmptyDirs = settings.IncludeEmptyDirs + } switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreiface/options/unixfs.go b/core/coreiface/options/unixfs.go index 45e880ed1..32303ffc2 100644 --- a/core/coreiface/options/unixfs.go +++ b/core/coreiface/options/unixfs.go @@ -48,10 +48,12 @@ type UnixfsAddSettings struct { Silent bool Progress bool - PreserveMode bool - PreserveMtime bool - Mode os.FileMode - Mtime time.Time + PreserveMode bool + PreserveMtime bool + Mode os.FileMode + Mtime time.Time + IncludeEmptyDirs bool + IncludeEmptyDirsSet bool } type UnixfsLsSettings struct { @@ -93,10 +95,12 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Silent: false, Progress: false, - PreserveMode: false, - PreserveMtime: false, - Mode: 0, - Mtime: time.Time{}, + PreserveMode: false, + PreserveMtime: false, + Mode: 0, + Mtime: time.Time{}, + IncludeEmptyDirs: true, // default: include empty directories + IncludeEmptyDirsSet: false, } for _, opt := range opts { @@ -396,3 +400,12 @@ func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption { return nil } } + +// IncludeEmptyDirs tells the adder to include empty directories in the DAG +func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.IncludeEmptyDirs = include + settings.IncludeEmptyDirsSet = true + return nil + } +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 55a9d5bec..3b0bee057 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -26,6 +26,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" + "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/tracing" @@ -52,17 +53,18 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG bufferedDS := ipld.NewBufferedDAG(ctx, ds) return &Adder{ - ctx: ctx, - pinning: p, - gcLocker: bs, - dagService: ds, - bufferedDS: bufferedDS, - Progress: false, - Pin: true, - Trickle: false, - MaxLinks: ihelper.DefaultLinksPerBlock, - MaxHAMTFanout: uio.DefaultShardWidth, - Chunker: "", + ctx: ctx, + pinning: p, + gcLocker: bs, + dagService: ds, + bufferedDS: bufferedDS, + Progress: false, + Pin: true, + Trickle: false, + MaxLinks: ihelper.DefaultLinksPerBlock, + MaxHAMTFanout: uio.DefaultShardWidth, + Chunker: "", + IncludeEmptyDirs: config.DefaultUnixFSIncludeEmptyDirs, }, nil } @@ -91,10 +93,11 @@ type Adder struct { CidBuilder cid.Builder liveNodes uint64 - PreserveMode bool - PreserveMtime bool - FileMode os.FileMode - FileMtime time.Time + PreserveMode bool + PreserveMtime bool + FileMode os.FileMode + FileMtime time.Time + IncludeEmptyDirs bool } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -480,6 +483,24 @@ func (adder *Adder) addFile(path string, file files.File) error { func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) + // Peek at first entry to check if directory is empty. + // We advance the iterator once here and continue from this position + // in the processing loop below. This avoids allocating a slice to + // collect all entries just to check for emptiness. + it := dir.Entries() + hasEntry := it.Next() + if !hasEntry { + if err := it.Err(); err != nil { + return err + } + // Directory is empty. Skip it unless IncludeEmptyDirs is set or + // this is the toplevel directory (we always include the root). + if !adder.IncludeEmptyDirs && !toplevel { + log.Debugf("skipping empty directory: %s", path) + return nil + } + } + // if we need to store mode or modification time then create a new root which includes that data if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil, @@ -515,13 +536,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory } } - it := dir.Entries() - for it.Next() { + // Process directory entries. The iterator was already advanced once above + // to peek for emptiness, so we start from that position. + for hasEntry { fpath := gopath.Join(path, it.Name()) - err := adder.addFileNode(ctx, fpath, it.Node(), false) - if err != nil { + if err := adder.addFileNode(ctx, fpath, it.Node(), false); err != nil { return err } + hasEntry = it.Next() } return it.Err() diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 524852a30..ef8f9df76 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -10,6 +10,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [๐Ÿ”ข UnixFS CID Profiles (IPIP-499)](#-unixfs-cid-profiles-ipip-499) - [๐Ÿงน Automatic cleanup of interrupted imports](#-automatic-cleanup-of-interrupted-imports) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - [Track total size when adding pins](#track-total-size-when-adding-pins) @@ -30,6 +31,43 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. ### ๐Ÿ”ฆ Highlights +#### ๐Ÿ”ข UnixFS CID Profiles (IPIP-499) + +This release introduces [IPIP-499](https://github.com/ipfs/specs/pull/499) UnixFS CID Profiles for reproducible CID generation across IPFS implementations. + +**New Profiles** + +- `unixfs-v1-2025`: the recommended profile for CIDv1 imports with deterministic HAMT directory sharding based on block size estimation +- `unixfs-v0-2015` (alias `legacy-cid-v0`): preserves legacy CIDv0 behavior for backward compatibility + +Apply a profile with: `ipfs config profile apply unixfs-v1-2025` + +**New `Import.*` Configuration Options** + +New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options allow fine-grained control over import parameters: + +- `Import.CidVersion`: CID version (0 or 1) +- `Import.HashFunction`: hash algorithm +- `Import.UnixFSChunker`: chunking strategy +- `Import.UnixFSRawLeaves`: raw leaf blocks +- `Import.UnixFSFileMaxLinks`: max children per file node +- `Import.UnixFSDirectoryMaxLinks`: max children per basic directory +- `Import.UnixFSHAMTDirectoryMaxFanout`: HAMT shard width +- `Import.UnixFSHAMTDirectorySizeThreshold`: threshold for HAMT sharding +- `Import.UnixFSHAMTDirectorySizeEstimation`: estimation mode (`links`, `block`, or `disabled`) +- `Import.UnixFSDAGLayout`: DAG layout (`balanced` or `trickle`) + +**Deprecated Profiles** + +The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` for CIDv1 imports. + +**CLI Changes** + +- New `--dereference-symlinks` flag for `ipfs add` recursively resolves symlinks to their target content (replaces deprecated `--dereference-args` which only worked on CLI arguments) +- New `--empty-dirs` / `-E` flag for `ipfs add` controls inclusion of empty directories (default: true) +- New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false) +- The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default + #### ๐Ÿงน Automatic cleanup of interrupted imports If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection. diff --git a/docs/config.md b/docs/config.md index 16987b0ce..3b040c61e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -243,7 +243,6 @@ config file at runtime. - [`Import.UnixFSHAMTDirectoryMaxFanout`](#importunixfshamtdirectorymaxfanout) - [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) - [`Import.UnixFSHAMTDirectorySizeEstimation`](#importunixfshamtdirectorysizeestimation) - - [`Import.UnixFSSymlinkMode`](#importunixfssymlinkmode) - [`Import.UnixFSDAGLayout`](#importunixfsdaglayout) - [`Version`](#version) - [`Version.AgentSuffix`](#versionagentsuffix) @@ -3642,9 +3641,11 @@ Type: `flag` ## `Import` -Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. +Options to configure the default parameters used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. -Note that using flags will override the options defined here. +These options implement [IPIP-499: UnixFS CID Profiles](https://github.com/ipfs/specs/pull/499) for reproducible CID generation across IPFS implementations. Instead of configuring individual options, you can apply a predefined profile with `ipfs config profile apply `. See [Profiles](#profiles) for available options like `unixfs-v1-2025`. + +Note that using CLI flags will override the options defined here. ### `Import.CidVersion` @@ -3845,21 +3846,6 @@ Default: `links` Type: `optionalString` -### `Import.UnixFSSymlinkMode` - -Controls how symbolic links are handled during import. - -Accepted values: - -- `preserve` (default): Store symlinks as UnixFS symlink nodes containing the target path. -- `dereference`: Follow symlinks and import the target content instead. - -Commands affected: `ipfs add` - -Default: `preserve` - -Type: `optionalString` - ### `Import.UnixFSDAGLayout` Controls the DAG layout used when chunking files. @@ -3918,7 +3904,7 @@ applied with the `--profile` flag to `ipfs init` or with the `ipfs config profil apply` command. When a profile is applied a backup of the configuration file will be created in `$IPFS_PATH`. -Configuration profiles can be applied additively. For example, both the `test-cid-v1` and `lowpower` profiles can be applied one after the other. +Configuration profiles can be applied additively. For example, both the `unixfs-v1-2025` and `lowpower` profiles can be applied one after the other. The available configuration profiles are listed below. You can also find them documented in `ipfs config profile --help`. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 234df7d5d..7db9fd2a1 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -84,7 +84,7 @@ require ( github.com/ipfs/go-ds-pebble v0.5.8 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f998634d5..18db9cc6d 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -301,8 +301,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/go.mod b/go.mod index 37ceedbf7..9a7b844a8 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.8 github.com/ipfs/go-fs-lock v0.1.1 - github.com/ipfs/go-ipfs-cmds v0.15.0 + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 4e1c346a1..ee5503567 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/test/cli/add_test.go b/test/cli/add_test.go index cda0c977d..9a232112e 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -449,6 +449,186 @@ func TestAdd(t *testing.T) { require.Equal(t, 992, len(root.Links)) }) }) + + t.Run("ipfs add --hidden", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with hidden file + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "hidden-test") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0644)) + return testDir + } + + t.Run("default excludes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.NotContains(t, lsOutput, ".hidden") + }) + + t.Run("--hidden includes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--hidden", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.Contains(t, lsOutput, ".hidden") + }) + + t.Run("-H includes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "-H", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.Contains(t, lsOutput, ".hidden") + }) + }) + + t.Run("ipfs add --empty-dirs", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with empty subdirectory + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "empty-dirs-test") + require.NoError(t, err) + require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0644)) + return testDir + } + + t.Run("default includes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") + }) + + t.Run("--empty-dirs=true includes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=true", testDir).Stdout.Trimmed() + require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") + }) + + t.Run("--empty-dirs=false excludes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=false", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.NotContains(t, lsOutput, "empty-subdir") + require.Contains(t, lsOutput, "file.txt") + }) + }) + + t.Run("ipfs add --dereference-symlinks", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with a file and symlink to it + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-test") + require.NoError(t, err) + + targetFile := filepath.Join(testDir, "target.txt") + require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0644)) + + // Create symlink pointing to target + require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt"))) + + return testDir + } + + t.Run("default preserves symlinks", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + // Add directory with symlink (default: preserve) + dirCID := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + + // Get to a new directory and verify symlink is preserved + outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") + require.NoError(t, err) + node.IPFS("get", "-o", outDir, dirCID) + + // Check that link.txt is a symlink (ipfs get -o puts files directly in outDir) + linkPath := filepath.Join(outDir, "link.txt") + fi, err := os.Lstat(linkPath) + require.NoError(t, err) + require.True(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be a symlink") + + // Verify symlink target + target, err := os.Readlink(linkPath) + require.NoError(t, err) + require.Equal(t, "target.txt", target) + }) + + t.Run("--dereference-symlinks resolves nested symlinks", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + // Add directory with dereference flag - nested symlinks should be resolved + dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed() + + // Get and verify symlink was dereferenced to regular file + outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") + require.NoError(t, err) + node.IPFS("get", "-o", outDir, dirCID) + + linkPath := filepath.Join(outDir, "link.txt") + fi, err := os.Lstat(linkPath) + require.NoError(t, err) + + // Should be a regular file, not a symlink + require.False(t, fi.Mode()&os.ModeSymlink != 0, + "link.txt should be dereferenced to regular file, not preserved as symlink") + + // Content should match the target file + content, err := os.ReadFile(linkPath) + require.NoError(t, err) + require.Equal(t, "target content", string(content)) + }) + + t.Run("--dereference-args is deprecated", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + res := node.RunIPFS("add", "-Q", "--dereference-args", filepath.Join(testDir, "link.txt")) + require.Error(t, res.Err) + require.Contains(t, res.Stderr.String(), "--dereference-args is deprecated") + }) + }) } func TestAddFastProvide(t *testing.T) { diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 234086a25..7c9419af4 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -141,7 +141,7 @@ require ( github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index c99ca0aa4..ebf3395c1 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -312,8 +312,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc= github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= From aaf05db536257ebed12e491d58e5bba403d73289 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 19 Jan 2026 06:13:13 +0100 Subject: [PATCH 03/22] test(add): add CID profile tests and wire SizeEstimationMode add comprehensive test suite for UnixFS CID determinism per IPIP-499: - verify exact HAMT threshold boundary for both estimation modes: - v0-2015 (links): sum(name_len + cid_len) == 262144 - v1-2025 (block): serialized block size == 262144 - verify HAMT triggers at threshold + 1 byte for both profiles - add all deterministic CIDs for cross-implementation testing also wires SizeEstimationMode through CLI/API, allowing Import.UnixFSHAMTSizeEstimation config to take effect. bumps boxo to ipfs/boxo@6707376 which aligns HAMT threshold with JS implementation (uses > instead of >=), fixing CID determinism at the exact 256 KiB boundary. --- core/commands/add.go | 8 + core/coreapi/unixfs.go | 3 + core/coreiface/options/unixfs.go | 31 +- core/coreunix/add.go | 89 ++-- docs/changelogs/v0.40.md | 4 + docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/cli/add_test.go | 255 +++++------ test/cli/cid_profiles_test.go | 592 +++++++++++++++++++++++++ test/cli/harness/ipfs.go | 14 +- test/cli/testutils/protobuf.go | 39 ++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 15 files changed, 842 insertions(+), 211 deletions(-) create mode 100644 test/cli/cid_profiles_test.go create mode 100644 test/cli/testutils/protobuf.go diff --git a/core/commands/add.go b/core/commands/add.go index a56b69434..703213add 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -15,6 +15,7 @@ import ( "github.com/cheggaaa/pb" "github.com/ipfs/boxo/files" + uio "github.com/ipfs/boxo/ipld/unixfs/io" mfs "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/verifcid" @@ -300,6 +301,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import maxFileLinks, maxFileLinksSet := req.Options[maxFileLinksOptionName].(int) maxDirectoryLinks, maxDirectoryLinksSet := req.Options[maxDirectoryLinksOptionName].(int) maxHAMTFanout, maxHAMTFanoutSet := req.Options[maxHAMTFanoutOptionName].(int) + var sizeEstimationMode uio.SizeEstimationMode nocopy, _ := req.Options[noCopyOptionName].(bool) fscache, _ := req.Options[fstoreCacheOptionName].(bool) cidVer, cidVerSet := req.Options[cidVersionOptionName].(int) @@ -376,6 +378,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)) } + // SizeEstimationMode is always set from config (no CLI flag) + sizeEstimationMode = cfg.Import.HAMTSizeEstimationMode() + fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot) fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait) @@ -471,6 +476,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import opts = append(opts, options.Unixfs.MaxHAMTFanout(maxHAMTFanout)) } + // SizeEstimationMode is always set from config + opts = append(opts, options.Unixfs.SizeEstimationMode(sizeEstimationMode)) + if trickle { opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index a5507f557..729b4851a 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -177,6 +177,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options if settings.MaxHAMTFanoutSet { fileAdder.MaxHAMTFanout = settings.MaxHAMTFanout } + if settings.SizeEstimationModeSet { + fileAdder.SizeEstimationMode = settings.SizeEstimationMode + } fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix fileAdder.PreserveMode = settings.PreserveMode diff --git a/core/coreiface/options/unixfs.go b/core/coreiface/options/unixfs.go index 32303ffc2..9018ec351 100644 --- a/core/coreiface/options/unixfs.go +++ b/core/coreiface/options/unixfs.go @@ -24,16 +24,18 @@ type UnixfsAddSettings struct { CidVersion int MhType uint64 - Inline bool - InlineLimit int - RawLeaves bool - RawLeavesSet bool - MaxFileLinks int - MaxFileLinksSet bool - MaxDirectoryLinks int - MaxDirectoryLinksSet bool - MaxHAMTFanout int - MaxHAMTFanoutSet bool + Inline bool + InlineLimit int + RawLeaves bool + RawLeavesSet bool + MaxFileLinks int + MaxFileLinksSet bool + MaxDirectoryLinks int + MaxDirectoryLinksSet bool + MaxHAMTFanout int + MaxHAMTFanoutSet bool + SizeEstimationMode *io.SizeEstimationMode + SizeEstimationModeSet bool Chunker string Layout Layout @@ -239,6 +241,15 @@ func (unixfsOpts) MaxHAMTFanout(n int) UnixfsAddOption { } } +// SizeEstimationMode specifies how directory size is estimated for HAMT sharding decisions. +func (unixfsOpts) SizeEstimationMode(mode io.SizeEstimationMode) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.SizeEstimationMode = &mode + settings.SizeEstimationModeSet = true + return nil + } +} + // Inline tells the adder to inline small blocks into CIDs func (unixfsOpts) Inline(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 3b0bee057..49d2449af 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -70,28 +70,29 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG // Adder holds the switches passed to the `add` command. type Adder struct { - ctx context.Context - pinning pin.Pinner - gcLocker bstore.GCLocker - dagService ipld.DAGService - bufferedDS *ipld.BufferedDAG - Out chan<- interface{} - Progress bool - Pin bool - PinName string - Trickle bool - RawLeaves bool - MaxLinks int - MaxDirectoryLinks int - MaxHAMTFanout int - Silent bool - NoCopy bool - Chunker string - mroot *mfs.Root - unlocker bstore.Unlocker - tempRoot cid.Cid - CidBuilder cid.Builder - liveNodes uint64 + ctx context.Context + pinning pin.Pinner + gcLocker bstore.GCLocker + dagService ipld.DAGService + bufferedDS *ipld.BufferedDAG + Out chan<- interface{} + Progress bool + Pin bool + PinName string + Trickle bool + RawLeaves bool + MaxLinks int + MaxDirectoryLinks int + MaxHAMTFanout int + SizeEstimationMode *uio.SizeEstimationMode + Silent bool + NoCopy bool + Chunker string + mroot *mfs.Root + unlocker bstore.Unlocker + tempRoot cid.Cid + CidBuilder cid.Builder + liveNodes uint64 PreserveMode bool PreserveMtime bool @@ -107,9 +108,10 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) { // Note, this adds it to DAGService already. mr, err := mfs.NewEmptyRoot(adder.ctx, adder.dagService, nil, nil, mfs.MkdirOpts{ - CidBuilder: adder.CidBuilder, - MaxLinks: adder.MaxDirectoryLinks, - MaxHAMTFanout: adder.MaxHAMTFanout, + CidBuilder: adder.CidBuilder, + MaxLinks: adder.MaxDirectoryLinks, + MaxHAMTFanout: adder.MaxHAMTFanout, + SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return nil, err @@ -273,11 +275,12 @@ func (adder *Adder) addNode(node ipld.Node, path string) error { dir := gopath.Dir(path) if dir != "." { opts := mfs.MkdirOpts{ - Mkparents: true, - Flush: false, - CidBuilder: adder.CidBuilder, - MaxLinks: adder.MaxDirectoryLinks, - MaxHAMTFanout: adder.MaxHAMTFanout, + Mkparents: true, + Flush: false, + CidBuilder: adder.CidBuilder, + MaxLinks: adder.MaxDirectoryLinks, + MaxHAMTFanout: adder.MaxHAMTFanout, + SizeEstimationMode: adder.SizeEstimationMode, } if err := mfs.Mkdir(mr, dir, opts); err != nil { return err @@ -505,11 +508,12 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil, mfs.MkdirOpts{ - CidBuilder: adder.CidBuilder, - MaxLinks: adder.MaxDirectoryLinks, - MaxHAMTFanout: adder.MaxHAMTFanout, - ModTime: adder.FileMtime, - Mode: adder.FileMode, + CidBuilder: adder.CidBuilder, + MaxLinks: adder.MaxDirectoryLinks, + MaxHAMTFanout: adder.MaxHAMTFanout, + ModTime: adder.FileMtime, + Mode: adder.FileMode, + SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return err @@ -523,13 +527,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory return err } err = mfs.Mkdir(mr, path, mfs.MkdirOpts{ - Mkparents: true, - Flush: false, - CidBuilder: adder.CidBuilder, - Mode: adder.FileMode, - ModTime: adder.FileMtime, - MaxLinks: adder.MaxDirectoryLinks, - MaxHAMTFanout: adder.MaxHAMTFanout, + Mkparents: true, + Flush: false, + CidBuilder: adder.CidBuilder, + Mode: adder.FileMode, + ModTime: adder.FileMtime, + MaxLinks: adder.MaxDirectoryLinks, + MaxHAMTFanout: adder.MaxHAMTFanout, + SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return err diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index ef8f9df76..7da198390 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -68,6 +68,10 @@ The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs - New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false) - The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default +**HAMT Threshold Fix** + +The HAMT directory sharding threshold comparison was aligned with the JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). The comparison changed from `>=` to `>`, meaning a directory exactly at the 256 KiB threshold now stays as a basic (flat) directory instead of converting to HAMT. This is a subtle 1-byte boundary change that improves CID determinism across implementations. + #### ๐Ÿงน Automatic cleanup of interrupted imports If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 7db9fd2a1..6b86420d1 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 + github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 18db9cc6d..f4a4eae36 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 9a7b844a8..219c9a5dd 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 + github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index ee5503567..6f71ded27 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/cli/add_test.go b/test/cli/add_test.go index 9a232112e..00a26db4a 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/dustin/go-humanize" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" @@ -166,7 +165,7 @@ func TestAdd(t *testing.T) { // // UnixFSChunker=size-262144 (256KiB) // Import.UnixFSFileMaxLinks=174 - node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0") // legacy-cid-v0 for determinism across all params + node := harness.NewT(t).NewNode().Init("--profile=unixfs-v0-2015") // unixfs-v0-2015 for determinism across all params node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSChunker = *config.NewOptionalString("size-262144") // 256 KiB chunks cfg.Import.UnixFSFileMaxLinks = *config.NewOptionalInteger(174) // max 174 per level @@ -187,9 +186,9 @@ func TestAdd(t *testing.T) { require.Equal(t, "QmbBftNHWmjSWKLC49dMVrfnY8pjrJYntiAXirFJ7oJrNk", cidStr) }) - t.Run("ipfs init --profile=legacy-cid-v0 sets config that produces legacy CIDv0", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v0-2015 sets config that produces legacy CIDv0", func(t *testing.T) { t.Parallel() - node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0") + node := harness.NewT(t).NewNode().Init("--profile=unixfs-v0-2015") node.StartDaemon() defer node.StopDaemon() @@ -197,10 +196,10 @@ func TestAdd(t *testing.T) { require.Equal(t, shortStringCidV0, cidStr) }) - t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v0-2015 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks", func(t *testing.T) { t.Parallel() seed := "v0-seed" - profile := "--profile=legacy-cid-v0" + profile := "--profile=unixfs-v0-2015" t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) { t.Parallel() @@ -232,12 +231,15 @@ func TestAdd(t *testing.T) { }) }) - t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v0-2015 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { t.Parallel() - seed := "hamt-legacy-cid-v0" - profile := "--profile=legacy-cid-v0" + seed := "hamt-unixfs-v0-2015" + profile := "--profile=unixfs-v0-2015" - t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { + // unixfs-v0-2015 uses links-based estimation: size = sum(nameLen + cidLen) + // Threshold is 256KiB = 262144 bytes + + t.Run("at UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -246,18 +248,24 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV0Length, "255KiB", seed) + // Create directory exactly at the 256KiB threshold using links estimation. + // Links estimation: size = numFiles * (nameLen + cidLen) + // 4096 * (30 + 34) = 4096 * 64 = 262144 = threshold exactly + // With > comparison: stays as basic directory + // With >= comparison: converts to HAMT + const numFiles, nameLen = 4096, 30 + err = createDirectoryForHAMTLinksEstimation(randDir, cidV0Length, numFiles, nameLen, nameLen, seed) require.NoError(t, err) + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Confirm the number of links is more than UnixFSHAMTDirectorySizeThreshold (indicating regular "basic" directory" + // Should remain a basic directory (threshold uses > not >=) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) - require.Equal(t, 903, len(root.Links)) + require.Equal(t, numFiles, len(root.Links), "expected basic directory at exact threshold") }) - t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { + t.Run("over UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -266,21 +274,25 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV0Length, "257KiB", seed) + // Create directory just over the 256KiB threshold using links estimation. + // Links estimation: size = numFiles * (nameLen + cidLen) + // 4097 * (30 + 34) = 4097 * 64 = 262208 > 262144, exceeds threshold + const numFiles, nameLen = 4097, 30 + err = createDirectoryForHAMTLinksEstimation(randDir, cidV0Length, numFiles, nameLen, nameLen, seed) require.NoError(t, err) + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Confirm this time, the number of links is less than UnixFSHAMTDirectorySizeThreshold + // Should be HAMT sharded (root links <= fanout of 256) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) - require.Equal(t, 252, len(root.Links)) + require.LessOrEqual(t, len(root.Links), 256, "expected HAMT directory when over threshold") }) }) - t.Run("ipfs init --profile=test-cid-v1 produces CIDv1 with raw leaves", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v1-2025 produces CIDv1 with raw leaves", func(t *testing.T) { t.Parallel() - node := harness.NewT(t).NewNode().Init("--profile=test-cid-v1") + node := harness.NewT(t).NewNode().Init("--profile=unixfs-v1-2025") node.StartDaemon() defer node.StopDaemon() @@ -288,105 +300,21 @@ func TestAdd(t *testing.T) { require.Equal(t, shortStringCidV1, cidStr) // raw leaf }) - t.Run("ipfs init --profile=test-cid-v1 applies UnixFSChunker=size-1048576", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v1-2025 applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=1024", func(t *testing.T) { t.Parallel() - seed := "v1-seed" - profile := "--profile=test-cid-v1" - - t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // Add 174MiB file: - // 174 * 1MiB should fit in single layer - cidStr := node.IPFSAddDeterministic("174MiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 174, len(root.Links)) - // expect same CID every time - require.Equal(t, "bafybeigwduxcf2aawppv3isnfeshnimkyplvw3hthxjhr2bdeje4tdaicu", cidStr) - }) - - t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer - cidStr := node.IPFSAddDeterministic("175MiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 2, len(root.Links)) - // expect same CID every time - require.Equal(t, "bafybeidhd7lo2n2v7lta5yamob3xwhbxcczmmtmhquwhjesi35jntf7mpu", cidStr) - }) - }) - - t.Run("ipfs init --profile=test-cid-v1 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { - t.Parallel() - seed := "hamt-cid-v1" - profile := "--profile=test-cid-v1" - - t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV1Length, "255KiB", seed) - require.NoError(t, err) - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory" - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 897, len(root.Links)) - }) - - t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV1Length, "257KiB", seed) - require.NoError(t, err) - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 252, len(root.Links)) - }) - }) - - t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=1024", func(t *testing.T) { - t.Parallel() - seed := "v1-seed-1024" - profile := "--profile=test-cid-v1-wide" + seed := "v1-2025-seed" + profile := "--profile=unixfs-v1-2025" t.Run("under UnixFSFileMaxLinks=1024", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() defer node.StopDaemon() - // Add 174MiB file: // 1024 * 1MiB should fit in single layer cidStr := node.IPFSAddDeterministic("1024MiB", seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 1024, len(root.Links)) - // expect same CID every time - require.Equal(t, "bafybeiej5w63ir64oxgkr5htqmlerh5k2rqflurn2howimexrlkae64xru", cidStr) }) t.Run("above UnixFSFileMaxLinks=1024", func(t *testing.T) { @@ -399,17 +327,19 @@ func TestAdd(t *testing.T) { root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 2, len(root.Links)) - // expect same CID every time - require.Equal(t, "bafybeieilp2qx24pe76hxrxe6bpef5meuxto3kj5dd6mhb5kplfeglskdm", cidStr) }) }) - t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) { + t.Run("ipfs init --profile=unixfs-v1-2025 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { t.Parallel() - seed := "hamt-cid-v1" - profile := "--profile=test-cid-v1-wide" + seed := "hamt-unixfs-v1-2025" + profile := "--profile=unixfs-v1-2025" - t.Run("under UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) { + // unixfs-v1-2025 uses block-based size estimation: size = sum(LinkSerializedSize) + // where LinkSerializedSize includes protobuf overhead (tags, varints, wrappers). + // Threshold is 256KiB = 262144 bytes + + t.Run("at UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -418,18 +348,25 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV1Length, "1023KiB", seed) + // Create directory exactly at the 256KiB threshold using block estimation. + // Block estimation: size = baseOverhead + numFiles * LinkSerializedSize + // LinkSerializedSize(11, 36, 0) = 55 bytes per link + // 4766 * 55 + 14 = 262130 + 14 = 262144 = threshold exactly + // With > comparison: stays as basic directory + // With >= comparison: converts to HAMT + const numFiles, nameLen = 4766, 11 + err = createDirectoryForHAMTBlockEstimation(randDir, cidV1Length, numFiles, nameLen, nameLen, seed) require.NoError(t, err) + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory" + // Should remain a basic directory (threshold uses > not >=) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) - require.Equal(t, 3599, len(root.Links)) + require.Equal(t, numFiles, len(root.Links), "expected basic directory at exact threshold") }) - t.Run("above UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) { + t.Run("over UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -438,15 +375,19 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total - err = createDirectoryForHAMT(randDir, cidV1Length, "1025KiB", seed) + // Create directory just over the 256KiB threshold using block estimation. + // Block estimation: size = baseOverhead + numFiles * LinkSerializedSize + // 4767 * 55 + 14 = 262185 + 14 = 262199 > 262144, exceeds threshold + const numFiles, nameLen = 4767, 11 + err = createDirectoryForHAMTBlockEstimation(randDir, cidV1Length, numFiles, nameLen, nameLen, seed) require.NoError(t, err) + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout + // Should be HAMT sharded (root links <= fanout of 256) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) - require.Equal(t, 992, len(root.Links)) + require.LessOrEqual(t, len(root.Links), 256, "expected HAMT directory when over threshold") }) }) @@ -807,30 +748,56 @@ func TestAddFastProvide(t *testing.T) { }) } -// createDirectoryForHAMT aims to create enough files with long names for the directory block to be close to the UnixFSHAMTDirectorySizeThreshold. -// The calculation is based on boxo's HAMTShardingSize and sizeBelowThreshold which calculates ballpark size of the block -// by adding length of link names and the binary cid length. -// See https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L491 -func createDirectoryForHAMT(dirPath string, cidLength int, unixfsNodeSizeTarget, seed string) error { - hamtThreshold, err := humanize.ParseBytes(unixfsNodeSizeTarget) - if err != nil { - return err - } +// createDirectoryForHAMTLinksEstimation creates a directory with the specified number +// of files using the links-based size estimation formula (size = numFiles * (nameLen + cidLen)). +// Used by legacy profiles (unixfs-v0-2015). +// +// Threshold behavior: boxo uses > comparison, so directory at exact threshold stays basic. +// Use DirBasicFiles for basic directory test, DirHAMTFiles for HAMT directory test. +// +// The lastNameLen parameter allows the last file to have a different name length, +// enabling exact +1 byte threshold tests. +// +// See boxo/ipld/unixfs/io/directory.go sizeBelowThreshold() for the links estimation. +func createDirectoryForHAMTLinksEstimation(dirPath string, cidLength, numFiles, nameLen, lastNameLen int, seed string) error { + return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) +} - // Calculate how many files with long filenames are needed to hit UnixFSHAMTDirectorySizeThreshold - nameLen := 255 // max that works across windows/macos/linux +// createDirectoryForHAMTBlockEstimation creates a directory with the specified number +// of files using the block-based size estimation formula (LinkSerializedSize with protobuf overhead). +// Used by modern profiles (unixfs-v1-2025). +// +// Threshold behavior: boxo uses > comparison, so directory at exact threshold stays basic. +// Use DirBasicFiles for basic directory test, DirHAMTFiles for HAMT directory test. +// +// The lastNameLen parameter allows the last file to have a different name length, +// enabling exact +1 byte threshold tests. +// +// See boxo/ipld/unixfs/io/directory.go estimatedBlockSize() for the block estimation. +func createDirectoryForHAMTBlockEstimation(dirPath string, cidLength, numFiles, nameLen, lastNameLen int, seed string) error { + return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) +} + +// createDeterministicFiles creates numFiles files with deterministic names. +// Files 0 to numFiles-2 have nameLen characters, and the last file has lastNameLen characters. +// Each file contains "x" (1 byte) for non-zero tsize in directory links. +func createDeterministicFiles(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { alphabetLen := len(testutils.AlphabetEasy) - numFiles := int(hamtThreshold) / (nameLen + cidLength) - // Deterministic pseudo-random bytes for static CID - drand, err := testutils.DeterministicRandomReader(unixfsNodeSizeTarget, seed) + // Deterministic pseudo-random bytes for static filenames + drand, err := testutils.DeterministicRandomReader("1MiB", seed) if err != nil { return err } - // Create necessary files in a single, flat directory for i := 0; i < numFiles; i++ { - buf := make([]byte, nameLen) + // Use lastNameLen for the final file + currentNameLen := nameLen + if i == numFiles-1 { + currentNameLen = lastNameLen + } + + buf := make([]byte, currentNameLen) _, err := io.ReadFull(drand, buf) if err != nil { return err @@ -838,21 +805,17 @@ func createDirectoryForHAMT(dirPath string, cidLength int, unixfsNodeSizeTarget, // Convert deterministic pseudo-random bytes to ASCII var sb strings.Builder - for _, b := range buf { - // Map byte to printable ASCII range (33-126) char := testutils.AlphabetEasy[int(b)%alphabetLen] sb.WriteRune(char) } - filename := sb.String()[:nameLen] + filename := sb.String()[:currentNameLen] filePath := filepath.Join(dirPath, filename) - // Create empty file - f, err := os.Create(filePath) - if err != nil { + // Create file with 1-byte content for non-zero tsize + if err := os.WriteFile(filePath, []byte("x"), 0644); err != nil { return err } - f.Close() } return nil } diff --git a/test/cli/cid_profiles_test.go b/test/cli/cid_profiles_test.go new file mode 100644 index 000000000..4f153bf48 --- /dev/null +++ b/test/cli/cid_profiles_test.go @@ -0,0 +1,592 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/ipfs/kubo/test/cli/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// cidProfileExpectations defines expected behaviors for a UnixFS import profile. +// This allows DRY testing of multiple profiles with the same test logic. +type cidProfileExpectations struct { + // Profile identification + Name string // canonical profile name from IPIP-499 + ProfileArgs []string // args to pass to ipfs init (empty for default behavior) + + // CID format expectations + CIDVersion int // 0 or 1 + HashFunc string // e.g., "sha2-256" + RawLeaves bool // true = raw codec for small files, false = dag-pb wrapped + + // File chunking expectations + ChunkSize string // e.g., "1MiB" or "256KiB" + FileMaxLinks int // max links before DAG rebalancing + + // HAMT directory sharding expectations. + // Threshold behavior: boxo converts to HAMT when size > HAMTThreshold (not >=). + // This means a directory exactly at the threshold stays as a basic (flat) directory. + HAMTFanout int // max links per HAMT shard bucket (256) + HAMTThreshold int // sharding threshold in bytes (262144 = 256 KiB) + HAMTSizeEstimation string // "block" (protobuf size) or "links" (legacy name+cid) + + // Test vector parameters for threshold boundary tests. + // - DirBasic: size == threshold (stays basic) + // - DirHAMT: size > threshold (converts to HAMT) + // For block estimation, last filename length is adjusted to hit exact thresholds. + DirBasicNameLen int // filename length for basic directory (files 0 to N-2) + DirBasicLastNameLen int // filename length for last file (0 = same as DirBasicNameLen) + DirBasicFiles int // file count for basic directory (at exact threshold) + DirHAMTNameLen int // filename length for HAMT directory (files 0 to N-2) + DirHAMTLastNameLen int // filename length for last file (0 = same as DirHAMTNameLen) + DirHAMTFiles int // total file count for HAMT directory (over threshold) + + // Expected deterministic CIDs for test vectors + SmallFileCID string // CID for single byte "x" + FileAtMaxLinksCID string // CID for file at max links + FileOverMaxLinksCID string // CID for file triggering rebalance + DirBasicCID string // CID for basic directory (at exact threshold, stays flat) + DirHAMTCID string // CID for HAMT directory (over threshold, sharded) +} + +// unixfsV02015 is the legacy profile for backward-compatible CID generation. +// Alias: legacy-cid-v0 +var unixfsV02015 = cidProfileExpectations{ + Name: "unixfs-v0-2015", + ProfileArgs: []string{"--profile=unixfs-v0-2015"}, + + CIDVersion: 0, + HashFunc: "sha2-256", + RawLeaves: false, + + ChunkSize: "256KiB", + FileMaxLinks: 174, + + HAMTFanout: 256, + HAMTThreshold: 262144, // 256 KiB + HAMTSizeEstimation: "links", + DirBasicNameLen: 30, // 4096 * (30 + 34) = 262144 exactly at threshold + DirBasicFiles: 4096, // 4096 * 64 = 262144 (stays basic with >) + DirHAMTNameLen: 31, // 4033 * (31 + 34) = 262145 exactly +1 over threshold + DirHAMTLastNameLen: 0, // 0 = same as DirHAMTNameLen (uniform filenames) + DirHAMTFiles: 4033, // 4033 * 65 = 262145 (becomes HAMT) + + SmallFileCID: "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", // "hello world" dag-pb wrapped + FileAtMaxLinksCID: "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", // 44544KiB with seed "v0-seed" + FileOverMaxLinksCID: "QmepeWtdmS1hHXx1oZXsPUv6bMrfRRKfZcoPPU4eEfjnbf", // 44800KiB with seed "v0-seed" + DirBasicCID: "QmX5GtRk3TSSEHtdrykgqm4eqMEn3n2XhfkFAis5fjyZmN", // 4096 files at threshold + DirHAMTCID: "QmeMiJzmhpJAUgynAcxTQYek5PPKgdv3qEvFsdV3XpVnvP", // 4033 files +1 over threshold +} + +// unixfsV12025 is the recommended profile for cross-implementation CID determinism. +var unixfsV12025 = cidProfileExpectations{ + Name: "unixfs-v1-2025", + ProfileArgs: []string{"--profile=unixfs-v1-2025"}, + + CIDVersion: 1, + HashFunc: "sha2-256", + RawLeaves: true, + + ChunkSize: "1MiB", + FileMaxLinks: 1024, + + HAMTFanout: 256, + HAMTThreshold: 262144, // 256 KiB + HAMTSizeEstimation: "block", + // Block size = numFiles * linkSize + 4 bytes overhead + // LinkSerializedSize(11, 36, 1) = 55, LinkSerializedSize(21, 36, 1) = 65, LinkSerializedSize(22, 36, 1) = 66 + DirBasicNameLen: 11, // 4765 files * 55 bytes + DirBasicLastNameLen: 21, // last file: 65 bytes; total: 4765*55 + 65 + 4 = 262144 (at threshold) + DirBasicFiles: 4766, // stays basic with > comparison + DirHAMTNameLen: 11, // 4765 files * 55 bytes + DirHAMTLastNameLen: 22, // last file: 66 bytes; total: 4765*55 + 66 + 4 = 262145 (+1 over threshold) + DirHAMTFiles: 4766, // becomes HAMT + + SmallFileCID: "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", // "hello world" raw leaf + FileAtMaxLinksCID: "bafybeihmf37wcuvtx4hpu7he5zl5qaf2ineo2lqlfrapokkm5zzw7zyhvm", // 1024MiB with seed "v1-2025-seed" + FileOverMaxLinksCID: "bafybeihmzokxxjqwxjcryerhp5ezpcog2wcawfryb2xm64xiakgm4a5jue", // 1025MiB with seed "v1-2025-seed" + DirBasicCID: "bafybeic3h7rwruealwxkacabdy45jivq2crwz6bufb5ljwupn36gicplx4", // 4766 files at 262144 bytes (threshold) + DirHAMTCID: "bafybeiegvuterwurhdtkikfhbxcldohmxp566vpjdofhzmnhv6o4freidu", // 4766 files at 262145 bytes (+1 over) +} + +// defaultProfile points to the profile that matches Kubo's implicit default behavior. +// Today this is unixfs-v0-2015. When Kubo changes defaults, update this pointer. +var defaultProfile = unixfsV02015 + +const ( + cidV0Length = 34 // CIDv0 sha2-256 + cidV1Length = 36 // CIDv1 sha2-256 +) + +// TestCIDProfiles generates deterministic test vectors for CID profile verification. +// Set CID_PROFILES_CAR_OUTPUT environment variable to export CAR files. +// Example: CID_PROFILES_CAR_OUTPUT=/tmp/cid-profiles go test -run TestCIDProfiles -v +func TestCIDProfiles(t *testing.T) { + t.Parallel() + + carOutputDir := os.Getenv("CID_PROFILES_CAR_OUTPUT") + exportCARs := carOutputDir != "" + if exportCARs { + if err := os.MkdirAll(carOutputDir, 0755); err != nil { + t.Fatalf("failed to create CAR output directory: %v", err) + } + t.Logf("CAR export enabled, writing to: %s", carOutputDir) + } + + // Test both IPIP-499 profiles + for _, profile := range []cidProfileExpectations{unixfsV02015, unixfsV12025} { + t.Run(profile.Name, func(t *testing.T) { + t.Parallel() + runProfileTests(t, profile, carOutputDir, exportCARs) + }) + } + + // Test default behavior (no profile specified) + t.Run("default", func(t *testing.T) { + t.Parallel() + // Default behavior should match defaultProfile (currently unixfs-v0-2015) + defaultExp := defaultProfile + defaultExp.Name = "default" + defaultExp.ProfileArgs = nil // no profile args = default behavior + runProfileTests(t, defaultExp, carOutputDir, exportCARs) + }) +} + +// runProfileTests runs all test vectors for a given profile. +func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir string, exportCARs bool) { + cidLen := cidV0Length + if exp.CIDVersion == 1 { + cidLen = cidV1Length + } + + t.Run("small-file", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // Use "hello world" for determinism - matches CIDs in add_test.go + cidStr := node.IPFSAddStr("hello world") + + // Verify CID version + verifyCIDVersion(t, node, cidStr, exp.CIDVersion) + + // Verify hash function + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + // Verify raw leaves vs wrapped + verifyRawLeaves(t, node, cidStr, exp.RawLeaves) + + // Verify deterministic CID if expected + if exp.SmallFileCID != "" { + require.Equal(t, exp.SmallFileCID, cidStr, "expected deterministic CID for small file") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_small-file.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + t.Run("file-at-max-links", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // Calculate file size: maxLinks * chunkSize + fileSize := fileAtMaxLinksSize(exp) + // Seed matches add_test.go for deterministic CIDs + seed := seedForProfile(exp) + cidStr := node.IPFSAddDeterministic(fileSize, seed) + + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.Equal(t, exp.FileMaxLinks, len(root.Links), + "expected exactly %d links at max", exp.FileMaxLinks) + + // Verify hash function on root + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + // Verify deterministic CID if expected + if exp.FileAtMaxLinksCID != "" { + require.Equal(t, exp.FileAtMaxLinksCID, cidStr, "expected deterministic CID for file at max links") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-max-links.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + t.Run("file-over-max-links-rebalanced", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // One more chunk triggers rebalancing + fileSize := fileOverMaxLinksSize(exp) + // Seed matches add_test.go for deterministic CIDs + seed := seedForProfile(exp) + cidStr := node.IPFSAddDeterministic(fileSize, seed) + + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.Equal(t, 2, len(root.Links), "expected 2 links after DAG rebalancing") + + // Verify hash function on root + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + // Verify deterministic CID if expected + if exp.FileOverMaxLinksCID != "" { + require.Equal(t, exp.FileOverMaxLinksCID, cidStr, "expected deterministic CID for rebalanced file") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-max-links.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + t.Run("dir-basic", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // Use consistent seed for deterministic CIDs + seed := hamtSeedForProfile(exp) + randDir, err := os.MkdirTemp(node.Dir, seed) + require.NoError(t, err) + + // Create basic (flat) directory exactly at threshold. + // With > comparison, directory at exact threshold stays basic. + basicLastNameLen := exp.DirBasicLastNameLen + if basicLastNameLen == 0 { + basicLastNameLen = exp.DirBasicNameLen + } + if exp.HAMTSizeEstimation == "block" { + err = createDirectoryForHAMTBlockEstimation(randDir, cidLen, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) + } else { + err = createDirectoryForHAMTLinksEstimation(randDir, cidLen, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) + } + require.NoError(t, err) + + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.Equal(t, exp.DirBasicFiles, len(root.Links), + "expected basic directory with %d links", exp.DirBasicFiles) + + // Verify hash function + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + // Verify size is exactly at threshold + if exp.HAMTSizeEstimation == "block" { + // Block estimation: verify actual serialized block size + blockSize := getBlockSize(t, node, cidStr) + require.Equal(t, exp.HAMTThreshold, blockSize, + "expected basic directory block size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, blockSize) + } + if exp.HAMTSizeEstimation == "links" { + // Links estimation: verify sum of (name_len + cid_len) for all links + linksSize := 0 + for _, link := range root.Links { + linksSize += len(link.Name) + cidLen + } + require.Equal(t, exp.HAMTThreshold, linksSize, + "expected basic directory links size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, linksSize) + } + + // Verify deterministic CID + if exp.DirBasicCID != "" { + require.Equal(t, exp.DirBasicCID, cidStr, "expected deterministic CID for basic directory") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_dir-basic.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s (%d files) -> %s", cidStr, exp.DirBasicFiles, carPath) + } + }) + + t.Run("dir-hamt", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // Use consistent seed for deterministic CIDs + seed := hamtSeedForProfile(exp) + randDir, err := os.MkdirTemp(node.Dir, seed) + require.NoError(t, err) + + // Create HAMT (sharded) directory exactly +1 byte over threshold. + // With > comparison, directory over threshold becomes HAMT. + lastNameLen := exp.DirHAMTLastNameLen + if lastNameLen == 0 { + lastNameLen = exp.DirHAMTNameLen + } + if exp.HAMTSizeEstimation == "block" { + err = createDirectoryForHAMTBlockEstimation(randDir, cidLen, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) + } else { + err = createDirectoryForHAMTLinksEstimation(randDir, cidLen, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) + } + require.NoError(t, err) + + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, + "expected HAMT directory with <=%d links", exp.HAMTFanout) + + // Verify hash function + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + // Verify deterministic CID + if exp.DirHAMTCID != "" { + require.Equal(t, exp.DirHAMTCID, cidStr, "expected deterministic CID for HAMT directory") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_dir-hamt.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s (%d files, HAMT root links: %d) -> %s", + cidStr, exp.DirHAMTFiles, len(root.Links), carPath) + } + }) +} + +// verifyCIDVersion checks that the CID has the expected version. +func verifyCIDVersion(t *testing.T, _ *harness.Node, cidStr string, expectedVersion int) { + t.Helper() + if expectedVersion == 0 { + require.True(t, strings.HasPrefix(cidStr, "Qm"), + "expected CIDv0 (starts with Qm), got: %s", cidStr) + } else { + require.True(t, strings.HasPrefix(cidStr, "b"), + "expected CIDv1 (base32, starts with b), got: %s", cidStr) + } +} + +// verifyHashFunction checks that the CID uses the expected hash function. +func verifyHashFunction(t *testing.T, node *harness.Node, cidStr, expectedHash string) { + t.Helper() + // Use ipfs cid format to get hash function info + // Format string %h gives the hash function name + res := node.IPFS("cid", "format", "-f", "%h", cidStr) + hashFunc := strings.TrimSpace(res.Stdout.String()) + require.Equal(t, expectedHash, hashFunc, + "expected hash function %s, got %s for CID %s", expectedHash, hashFunc, cidStr) +} + +// verifyRawLeaves checks whether the CID represents a raw leaf or dag-pb wrapped block. +// For CIDv1: raw leaves have codec 0x55 (raw), wrapped have codec 0x70 (dag-pb). +// For CIDv0: always dag-pb (no raw leaves possible). +func verifyRawLeaves(t *testing.T, node *harness.Node, cidStr string, expectRaw bool) { + t.Helper() + // Use ipfs cid format to get codec info + // Format string %c gives the codec name + res := node.IPFS("cid", "format", "-f", "%c", cidStr) + codec := strings.TrimSpace(res.Stdout.String()) + + if expectRaw { + require.Equal(t, "raw", codec, + "expected raw codec for raw leaves, got %s for CID %s", codec, cidStr) + } else { + require.Equal(t, "dag-pb", codec, + "expected dag-pb codec for wrapped leaves, got %s for CID %s", codec, cidStr) + } +} + +// getBlockSize returns the size of a block in bytes using ipfs block stat. +func getBlockSize(t *testing.T, node *harness.Node, cidStr string) int { + t.Helper() + res := node.IPFS("block", "stat", "--enc=json", cidStr) + var stat struct { + Size int `json:"Size"` + } + require.NoError(t, json.Unmarshal(res.Stdout.Bytes(), &stat)) + return stat.Size +} + +// fileAtMaxLinksSize returns the file size that produces exactly FileMaxLinks chunks. +func fileAtMaxLinksSize(exp cidProfileExpectations) string { + switch exp.ChunkSize { + case "1MiB": + return strings.Replace(exp.ChunkSize, "1MiB", "", 1) + + string(rune('0'+exp.FileMaxLinks/1000)) + + string(rune('0'+(exp.FileMaxLinks%1000)/100)) + + string(rune('0'+(exp.FileMaxLinks%100)/10)) + + string(rune('0'+exp.FileMaxLinks%10)) + "MiB" + case "256KiB": + // 174 * 256 KiB = 44544 KiB + totalKiB := exp.FileMaxLinks * 256 + return intToStr(totalKiB) + "KiB" + default: + panic("unknown chunk size: " + exp.ChunkSize) + } +} + +// fileOverMaxLinksSize returns the file size that triggers DAG rebalancing. +func fileOverMaxLinksSize(exp cidProfileExpectations) string { + switch exp.ChunkSize { + case "1MiB": + return intToStr(exp.FileMaxLinks+1) + "MiB" + case "256KiB": + // (174 + 1) * 256 KiB = 44800 KiB + totalKiB := (exp.FileMaxLinks + 1) * 256 + return intToStr(totalKiB) + "KiB" + default: + panic("unknown chunk size: " + exp.ChunkSize) + } +} + +func intToStr(n int) string { + if n == 0 { + return "0" + } + var digits []byte + for n > 0 { + digits = append([]byte{byte('0' + n%10)}, digits...) + n /= 10 + } + return string(digits) +} + +// seedForProfile returns the deterministic seed used in add_test.go for file tests. +func seedForProfile(exp cidProfileExpectations) string { + switch exp.Name { + case "unixfs-v0-2015", "default": + return "v0-seed" + case "unixfs-v1-2025": + return "v1-2025-seed" + default: + return exp.Name + "-seed" + } +} + +// hamtSeedForProfile returns the deterministic seed for HAMT directory tests. +// Uses the same seed for both under/at threshold tests to ensure consistency. +func hamtSeedForProfile(exp cidProfileExpectations) string { + switch exp.Name { + case "unixfs-v0-2015", "default": + return "hamt-unixfs-v0-2015" + case "unixfs-v1-2025": + return "hamt-unixfs-v1-2025" + default: + return "hamt-" + exp.Name + } +} + +// TestDefaultMatchesExpectedProfile verifies that default ipfs add behavior +// matches the expected profile (currently unixfs-v0-2015). +func TestDefaultMatchesExpectedProfile(t *testing.T) { + t.Parallel() + + node := harness.NewT(t).NewNode().Init() + node.StartDaemon() + defer node.StopDaemon() + + // Small file test + cidDefault := node.IPFSAddStr("x") + + // Same file with explicit profile + nodeWithProfile := harness.NewT(t).NewNode().Init(defaultProfile.ProfileArgs...) + nodeWithProfile.StartDaemon() + defer nodeWithProfile.StopDaemon() + + cidWithProfile := nodeWithProfile.IPFSAddStr("x") + + require.Equal(t, cidWithProfile, cidDefault, + "default behavior should match %s profile", defaultProfile.Name) +} + +// TestProtobufHelpers verifies the protobuf size calculation helpers. +func TestProtobufHelpers(t *testing.T) { + t.Parallel() + + t.Run("VarintLen", func(t *testing.T) { + // Varint encoding: 7 bits per byte, MSB indicates continuation + cases := []struct { + value uint64 + expected int + }{ + {0, 1}, + {127, 1}, // 0x7F - max 1-byte varint + {128, 2}, // 0x80 - min 2-byte varint + {16383, 2}, // 0x3FFF - max 2-byte varint + {16384, 3}, // 0x4000 - min 3-byte varint + {2097151, 3}, // 0x1FFFFF - max 3-byte varint + {2097152, 4}, // 0x200000 - min 4-byte varint + {268435455, 4}, // 0xFFFFFFF - max 4-byte varint + {268435456, 5}, // 0x10000000 - min 5-byte varint + {34359738367, 5}, // 0x7FFFFFFFF - max 5-byte varint + } + + for _, tc := range cases { + got := testutils.VarintLen(tc.value) + require.Equal(t, tc.expected, got, "VarintLen(%d)", tc.value) + } + }) + + t.Run("LinkSerializedSize", func(t *testing.T) { + // Test typical cases for directory links + cases := []struct { + nameLen int + cidLen int + tsize uint64 + expected int + }{ + // 255-char name, CIDv0 (34 bytes), tsize=0 + // Inner: 1+1+34 + 1+2+255 + 1+1 = 296 + // Outer: 1 + 2 + 296 = 299 + {255, 34, 0, 299}, + // 255-char name, CIDv1 (36 bytes), tsize=0 + // Inner: 1+1+36 + 1+2+255 + 1+1 = 298 + // Outer: 1 + 2 + 298 = 301 + {255, 36, 0, 301}, + // Short name (10 chars), CIDv1, tsize=0 + // Inner: 1+1+36 + 1+1+10 + 1+1 = 52 + // Outer: 1 + 1 + 52 = 54 + {10, 36, 0, 54}, + // 255-char name, CIDv1, large tsize + // Inner: 1+1+36 + 1+2+255 + 1+5 = 302 (tsize uses 5-byte varint) + // Outer: 1 + 2 + 302 = 305 + {255, 36, 34359738367, 305}, + } + + for _, tc := range cases { + got := testutils.LinkSerializedSize(tc.nameLen, tc.cidLen, tc.tsize) + require.Equal(t, tc.expected, got, "LinkSerializedSize(%d, %d, %d)", tc.nameLen, tc.cidLen, tc.tsize) + } + }) + + t.Run("EstimateFilesForBlockThreshold", func(t *testing.T) { + threshold := 262144 + nameLen := 255 + cidLen := 36 + var tsize uint64 = 0 + + numFiles := testutils.EstimateFilesForBlockThreshold(threshold, nameLen, cidLen, tsize) + require.Equal(t, 870, numFiles, "expected 870 files for threshold 262144") + + numFilesUnder := testutils.EstimateFilesForBlockThreshold(threshold-1, nameLen, cidLen, tsize) + require.Equal(t, 870, numFilesUnder, "expected 870 files for threshold 262143") + + numFilesOver := testutils.EstimateFilesForBlockThreshold(262185, nameLen, cidLen, tsize) + require.Equal(t, 871, numFilesOver, "expected 871 files for threshold 262185") + }) +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index 2f7a8f18e..1e08f8ed9 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "os" "reflect" "strings" @@ -148,9 +149,15 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro return res.Err } -/* -func (n *Node) IPFSDagExport(cid string, car *os.File) error { - log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, car.Name()) +// IPFSDagExport exports a DAG rooted at cid to a CAR file at carPath. +func (n *Node) IPFSDagExport(cid string, carPath string) error { + log.Debugf("node %d dag export of %s to %q", n.ID, cid, carPath) + car, err := os.Create(carPath) + if err != nil { + return err + } + defer car.Close() + res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"dag", "export", cid}, @@ -158,4 +165,3 @@ func (n *Node) IPFSDagExport(cid string, car *os.File) error { }) return res.Err } -*/ diff --git a/test/cli/testutils/protobuf.go b/test/cli/testutils/protobuf.go new file mode 100644 index 000000000..ea3cbd8d5 --- /dev/null +++ b/test/cli/testutils/protobuf.go @@ -0,0 +1,39 @@ +package testutils + +import "math/bits" + +// VarintLen returns the number of bytes needed to encode v as a protobuf varint. +func VarintLen(v uint64) int { + return int(9*uint32(bits.Len64(v))+64) / 64 +} + +// LinkSerializedSize calculates the serialized size of a single PBLink in a dag-pb block. +// This matches the calculation in boxo/ipld/unixfs/io/directory.go estimatedBlockSize(). +// +// The protobuf wire format for a PBLink is: +// +// PBNode.Links wrapper tag (1 byte) +// + varint length of inner message +// + Hash field: tag (1) + varint(cidLen) + cidLen +// + Name field: tag (1) + varint(nameLen) + nameLen +// + Tsize field: tag (1) + varint(tsize) +func LinkSerializedSize(nameLen, cidLen int, tsize uint64) int { + // Inner link message size + linkLen := 1 + VarintLen(uint64(cidLen)) + cidLen + // Hash field + 1 + VarintLen(uint64(nameLen)) + nameLen + // Name field + 1 + VarintLen(tsize) // Tsize field + + // Outer wrapper: tag (1 byte) + varint(linkLen) + linkLen + return 1 + VarintLen(uint64(linkLen)) + linkLen +} + +// EstimateFilesForBlockThreshold estimates how many files with given name/cid lengths +// will fit under the block size threshold. +// Returns the number of files that keeps the block size just under the threshold. +func EstimateFilesForBlockThreshold(threshold, nameLen, cidLen int, tsize uint64) int { + linkSize := LinkSerializedSize(nameLen, cidLen, tsize) + // Base overhead for empty directory node (Data field + minimal structure) + // Empirically determined to be 4 bytes for dag-pb directories + baseOverhead := 4 + return (threshold - baseOverhead) / linkSize +} diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 7c9419af4..1c8943e02 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 // indirect + github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index ebf3395c1..a89859cf4 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00 h1:e9p5CizXgzPlnxt1kzDyYNoKusO4cvDjNG33UqyVhwM= -github.com/ipfs/boxo v0.35.3-0.20260117004328-4ff72d072c00/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= +github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From e09310433993a164cfb24e7785ffada38b637393 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 20 Jan 2026 00:50:22 +0100 Subject: [PATCH 04/22] feat(add): --dereference-symlinks now resolves all symlinks Previously, resolving symlinks required two flags: - --dereference-args: resolved symlinks passed as CLI arguments - --dereference-symlinks: resolved symlinks inside directories Now --dereference-symlinks handles both cases. Users only need one flag to fully dereference symlinks when adding files to IPFS. The deprecated --dereference-args still works for backwards compatibility but is no longer necessary. --- core/commands/add.go | 14 ++-- docs/changelogs/v0.40.md | 2 +- docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 +- go.mod | 4 +- go.sum | 8 +- test/cli/add_test.go | 112 ++++++++++++++++++------- test/dependencies/go.mod | 4 +- test/dependencies/go.sum | 8 +- 9 files changed, 109 insertions(+), 55 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 703213add..102a410a9 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -157,9 +157,9 @@ target content instead: > ipfs add -r --dereference-symlinks ./mydir -This recursively resolves all symlinks encountered during directory traversal. -Symlinks to files become regular file content, symlinks to directories are -traversed and their contents are added. +This resolves all symlinks, including CLI arguments and those found inside +directories. Symlinks to files become regular file content, symlinks to +directories are traversed and their contents are added. CHUNKING EXAMPLES: @@ -331,11 +331,9 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) emptyDirs, _ := req.Options[emptyDirsOptionName].(bool) - // Handle --dereference-args deprecation - derefArgs, derefArgsSet := req.Options[cmds.DerefLong].(bool) - if derefArgsSet && derefArgs { - return fmt.Errorf("--dereference-args is deprecated: use --dereference-symlinks instead") - } + // Note: --dereference-args is deprecated but still works for backwards compatibility. + // The help text marks it as DEPRECATED. Users should use --dereference-symlinks instead, + // which is a superset (resolves both CLI arg symlinks AND nested symlinks in directories). // Wire --trickle from config if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() { diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 7da198390..f653bc611 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -63,7 +63,7 @@ The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs **CLI Changes** -- New `--dereference-symlinks` flag for `ipfs add` recursively resolves symlinks to their target content (replaces deprecated `--dereference-args` which only worked on CLI arguments) +- New `--dereference-symlinks` flag for `ipfs add` resolves all symlinks to their target content, both CLI arguments and symlinks inside directories. This is a superset of the deprecated `--dereference-args` flag which only resolved CLI argument symlinks. - New `--empty-dirs` / `-E` flag for `ipfs add` controls inclusion of empty directories (default: true) - New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false) - The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6b86420d1..e1d6032dc 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 + github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 @@ -84,7 +84,7 @@ require ( github.com/ipfs/go-ds-pebble v0.5.8 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f4a4eae36..0234fe5e0 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -301,8 +301,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 h1:M+5zwNetUgBTt2ywpX5QZ7PvIcvhz3Nw6pC7CMrLIQQ= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/go.mod b/go.mod index 219c9a5dd..df7b67898 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 + github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.8 github.com/ipfs/go-fs-lock v0.1.1 - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 6f71ded27..eaba02cde 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -372,8 +372,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 h1:M+5zwNetUgBTt2ywpX5QZ7PvIcvhz3Nw6pC7CMrLIQQ= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/test/cli/add_test.go b/test/cli/add_test.go index 00a26db4a..64945f405 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -485,20 +485,33 @@ func TestAdd(t *testing.T) { }) }) - t.Run("ipfs add --dereference-symlinks", func(t *testing.T) { + t.Run("ipfs add symlink handling", func(t *testing.T) { t.Parallel() - // Helper to create test directory with a file and symlink to it + // Helper to create test directory structure: + // testDir/ + // target.txt (file with "target content") + // link.txt -> target.txt (symlink at top level) + // subdir/ + // subsubdir/ + // nested-target.txt (file with "nested content") + // nested-link.txt -> nested-target.txt (symlink in sub-sub directory) setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-test") require.NoError(t, err) + // Top-level file and symlink targetFile := filepath.Join(testDir, "target.txt") require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0644)) - - // Create symlink pointing to target require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt"))) + // Nested file and symlink in sub-sub directory + subsubdir := filepath.Join(testDir, "subdir", "subsubdir") + require.NoError(t, os.MkdirAll(subsubdir, 0755)) + nestedTarget := filepath.Join(subsubdir, "nested-target.txt") + require.NoError(t, os.WriteFile(nestedTarget, []byte("nested content"), 0644)) + require.NoError(t, os.Symlink("nested-target.txt", filepath.Join(subsubdir, "nested-link.txt"))) + return testDir } @@ -512,62 +525,105 @@ func TestAdd(t *testing.T) { // Add directory with symlink (default: preserve) dirCID := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() - // Get to a new directory and verify symlink is preserved + // Get and verify symlinks are preserved outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") require.NoError(t, err) node.IPFS("get", "-o", outDir, dirCID) - // Check that link.txt is a symlink (ipfs get -o puts files directly in outDir) + // Check top-level symlink is preserved linkPath := filepath.Join(outDir, "link.txt") fi, err := os.Lstat(linkPath) require.NoError(t, err) require.True(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be a symlink") - - // Verify symlink target target, err := os.Readlink(linkPath) require.NoError(t, err) require.Equal(t, "target.txt", target) + + // Check nested symlink is preserved + nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") + fi, err = os.Lstat(nestedLinkPath) + require.NoError(t, err) + require.True(t, fi.Mode()&os.ModeSymlink != 0, "nested-link.txt should be a symlink") }) - t.Run("--dereference-symlinks resolves nested symlinks", func(t *testing.T) { + t.Run("--dereference-args only resolves CLI argument symlinks (deprecated)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) - // Add directory with dereference flag - nested symlinks should be resolved - dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed() + // Add a symlink directly as CLI argument with --dereference-args + // This should resolve the CLI arg symlink to the target file content + symlinkPath := filepath.Join(testDir, "link.txt") + targetPath := filepath.Join(testDir, "target.txt") - // Get and verify symlink was dereferenced to regular file - outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") + symlinkCID := node.IPFS("add", "-Q", "--dereference-args", symlinkPath).Stdout.Trimmed() + targetCID := node.IPFS("add", "-Q", targetPath).Stdout.Trimmed() + + // CIDs should match because --dereference-args resolves the symlink + require.Equal(t, targetCID, symlinkCID, + "--dereference-args should resolve CLI arg symlink to target content") + + // Now add the directory recursively with --dereference-args + // Nested symlinks should NOT be resolved (only CLI args are resolved) + dirCID := node.IPFS("add", "-r", "-Q", "--dereference-args", testDir).Stdout.Trimmed() + + outDir, err := os.MkdirTemp(node.Dir, "deref-args-out") require.NoError(t, err) node.IPFS("get", "-o", outDir, dirCID) - linkPath := filepath.Join(outDir, "link.txt") - fi, err := os.Lstat(linkPath) + // Nested symlink should still be a symlink (not dereferenced) + nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") + fi, err := os.Lstat(nestedLinkPath) require.NoError(t, err) - - // Should be a regular file, not a symlink - require.False(t, fi.Mode()&os.ModeSymlink != 0, - "link.txt should be dereferenced to regular file, not preserved as symlink") - - // Content should match the target file - content, err := os.ReadFile(linkPath) - require.NoError(t, err) - require.Equal(t, "target content", string(content)) + require.True(t, fi.Mode()&os.ModeSymlink != 0, + "--dereference-args should NOT resolve nested symlinks, only CLI args") }) - t.Run("--dereference-args is deprecated", func(t *testing.T) { + t.Run("--dereference-symlinks resolves ALL symlinks (superset of --dereference-args)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) - res := node.RunIPFS("add", "-Q", "--dereference-args", filepath.Join(testDir, "link.txt")) - require.Error(t, res.Err) - require.Contains(t, res.Stderr.String(), "--dereference-args is deprecated") + // Test 1: CLI argument symlink is resolved (like --dereference-args) + symlinkPath := filepath.Join(testDir, "link.txt") + targetPath := filepath.Join(testDir, "target.txt") + + symlinkCID := node.IPFS("add", "-Q", "--dereference-symlinks", symlinkPath).Stdout.Trimmed() + targetCID := node.IPFS("add", "-Q", targetPath).Stdout.Trimmed() + + require.Equal(t, targetCID, symlinkCID, + "--dereference-symlinks should resolve CLI arg symlink (like --dereference-args)") + + // Test 2: Nested symlinks in sub-sub directory are ALSO resolved + dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed() + + outDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-out") + require.NoError(t, err) + node.IPFS("get", "-o", outDir, dirCID) + + // Top-level symlink should be dereferenced to regular file + linkPath := filepath.Join(outDir, "link.txt") + fi, err := os.Lstat(linkPath) + require.NoError(t, err) + require.False(t, fi.Mode()&os.ModeSymlink != 0, + "link.txt should be dereferenced to regular file") + content, err := os.ReadFile(linkPath) + require.NoError(t, err) + require.Equal(t, "target content", string(content)) + + // Nested symlink in sub-sub directory should ALSO be dereferenced + nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") + fi, err = os.Lstat(nestedLinkPath) + require.NoError(t, err) + require.False(t, fi.Mode()&os.ModeSymlink != 0, + "nested-link.txt should be dereferenced (--dereference-symlinks resolves ALL symlinks)") + nestedContent, err := os.ReadFile(nestedLinkPath) + require.NoError(t, err) + require.Equal(t, "nested content", string(nestedContent)) }) }) } diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 1c8943e02..15decf41a 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,13 +135,13 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 // indirect + github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index a89859cf4..fecb136ab 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3 h1:X6iiSyBUwhKgQMzM57wSXVUZfivm5nWm5S/Y2SrSjhA= -github.com/ipfs/boxo v0.35.3-0.20260119043727-6707376002a3/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= +github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= @@ -312,8 +312,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc= github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 h1:M+5zwNetUgBTt2ywpX5QZ7PvIcvhz3Nw6pC7CMrLIQQ= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= From 088f1a74bc4fb37078de04850c1d756cc665d115 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 20 Jan 2026 02:47:50 +0100 Subject: [PATCH 05/22] chore: update boxo and improve changelog - update boxo to ebdaf07c (nil filter fix, thread-safety docs) - simplify changelog for IPIP-499 section - shorten test names, move context to comments --- docs/changelogs/v0.40.md | 38 +++++++++----------------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- test/cli/add_test.go | 15 +++++----- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +-- 8 files changed, 30 insertions(+), 41 deletions(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index f653bc611..73eda0e66 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -33,44 +33,32 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. #### ๐Ÿ”ข UnixFS CID Profiles (IPIP-499) -This release introduces [IPIP-499](https://github.com/ipfs/specs/pull/499) UnixFS CID Profiles for reproducible CID generation across IPFS implementations. +[IPIP-499](https://github.com/ipfs/specs/pull/499) CID Profiles are presets that pin down how files get split into blocks and organized into directories. Useful when you need the same CID for the same data across different software or versions. -**New Profiles** +**New configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles)** -- `unixfs-v1-2025`: the recommended profile for CIDv1 imports with deterministic HAMT directory sharding based on block size estimation -- `unixfs-v0-2015` (alias `legacy-cid-v0`): preserves legacy CIDv0 behavior for backward compatibility +- `unixfs-v1-2025`: modern CIDv1 profile with improved defaults +- `unixfs-v0-2015` (alias `legacy-cid-v0`): best-effort legacy CIDv0 behavior -Apply a profile with: `ipfs config profile apply unixfs-v1-2025` +Apply with: `ipfs config profile apply unixfs-v1-2025` -**New `Import.*` Configuration Options** +The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` or manually set specific `Import.*` settings instead. -New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options allow fine-grained control over import parameters: +**New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options** -- `Import.CidVersion`: CID version (0 or 1) -- `Import.HashFunction`: hash algorithm -- `Import.UnixFSChunker`: chunking strategy -- `Import.UnixFSRawLeaves`: raw leaf blocks -- `Import.UnixFSFileMaxLinks`: max children per file node -- `Import.UnixFSDirectoryMaxLinks`: max children per basic directory -- `Import.UnixFSHAMTDirectoryMaxFanout`: HAMT shard width -- `Import.UnixFSHAMTDirectorySizeThreshold`: threshold for HAMT sharding - `Import.UnixFSHAMTDirectorySizeEstimation`: estimation mode (`links`, `block`, or `disabled`) - `Import.UnixFSDAGLayout`: DAG layout (`balanced` or `trickle`) -**Deprecated Profiles** +**New [`ipfs add`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-add) CLI flags** -The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` for CIDv1 imports. - -**CLI Changes** - -- New `--dereference-symlinks` flag for `ipfs add` resolves all symlinks to their target content, both CLI arguments and symlinks inside directories. This is a superset of the deprecated `--dereference-args` flag which only resolved CLI argument symlinks. -- New `--empty-dirs` / `-E` flag for `ipfs add` controls inclusion of empty directories (default: true) -- New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false) -- The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default +- `--dereference-symlinks` resolves all symlinks to their target content, replacing the deprecated `--dereference-args` which only resolved CLI argument symlinks +- `--empty-dirs` / `-E` controls inclusion of empty directories (default: true) +- `--hidden` / `-H` includes hidden files (default: false) +- `--trickle` implicit default can be adjusted via `Import.UnixFSDAGLayout` **HAMT Threshold Fix** -The HAMT directory sharding threshold comparison was aligned with the JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). The comparison changed from `>=` to `>`, meaning a directory exactly at the 256 KiB threshold now stays as a basic (flat) directory instead of converting to HAMT. This is a subtle 1-byte boundary change that improves CID determinism across implementations. +HAMT directory sharding threshold changed from `>=` to `>` to match the JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. #### ๐Ÿงน Automatic cleanup of interrupted imports diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index e1d6032dc..e8fa8f91d 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 + github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 0234fe5e0..333078808 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index df7b67898..b73a38d5a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 + github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index eaba02cde..ef94ed09b 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/cli/add_test.go b/test/cli/add_test.go index 64945f405..03abb821d 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -546,15 +546,15 @@ func TestAdd(t *testing.T) { require.True(t, fi.Mode()&os.ModeSymlink != 0, "nested-link.txt should be a symlink") }) - t.Run("--dereference-args only resolves CLI argument symlinks (deprecated)", func(t *testing.T) { + // --dereference-args is deprecated but still works for backwards compatibility. + // It only resolves symlinks passed as CLI arguments, NOT symlinks found + // during directory traversal. Use --dereference-symlinks instead. + t.Run("--dereference-args resolves CLI args only", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) - - // Add a symlink directly as CLI argument with --dereference-args - // This should resolve the CLI arg symlink to the target file content symlinkPath := filepath.Join(testDir, "link.txt") targetPath := filepath.Join(testDir, "target.txt") @@ -581,14 +581,15 @@ func TestAdd(t *testing.T) { "--dereference-args should NOT resolve nested symlinks, only CLI args") }) - t.Run("--dereference-symlinks resolves ALL symlinks (superset of --dereference-args)", func(t *testing.T) { + // --dereference-symlinks resolves ALL symlinks: both CLI arguments AND + // symlinks found during directory traversal. This is a superset of + // the deprecated --dereference-args behavior. + t.Run("--dereference-symlinks resolves all symlinks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) - - // Test 1: CLI argument symlink is resolved (like --dereference-args) symlinkPath := filepath.Join(testDir, "link.txt") targetPath := filepath.Join(testDir, "target.txt") diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 15decf41a..83492ae30 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 // indirect + github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index fecb136ab..fbc1be308 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94 h1:QiqKd0sYY/GO1UML+bCU6tKrKv2cZCGLXdEQbzObX4Y= -github.com/ipfs/boxo v0.35.3-0.20260119230329-ffe1a9cc4a94/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= +github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 25ed654a37e3da0aa5ada410e1cd271a106ef284 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 22 Jan 2026 01:25:35 +0100 Subject: [PATCH 06/22] chore: update boxo to 5cf22196 --- docs/examples/kubo-as-a-library/go.mod | 8 ++++---- docs/examples/kubo-as-a-library/go.sum | 16 ++++++++-------- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- test/dependencies/go.mod | 4 ++-- test/dependencies/go.sum | 16 ++++++++-------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index e8fa8f91d..12908e7f0 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd + github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 @@ -86,7 +86,7 @@ require ( github.com/ipfs/go-fs-lock v0.1.1 // indirect github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect - github.com/ipfs/go-ipfs-pq v0.0.3 // indirect + github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect @@ -94,7 +94,7 @@ require ( github.com/ipfs/go-ipld-legacy v0.2.2 // indirect github.com/ipfs/go-log/v2 v2.9.0 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect - github.com/ipfs/go-peertaskqueue v0.8.2 // indirect + github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.2.3 // indirect github.com/ipfs/go-unixfsnode v1.10.2 // indirect github.com/ipld/go-car/v2 v2.16.0 // indirect @@ -114,7 +114,7 @@ require ( github.com/libp2p/go-doh-resolver v0.5.0 // indirect github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.36.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.37.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect github.com/libp2p/go-libp2p-pubsub v0.15.0 // indirect github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 333078808..73e6853eb 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -308,8 +308,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= -github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= -github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= +github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -327,8 +327,8 @@ github.com/ipfs/go-log/v2 v2.9.0 h1:l4b06AwVXwldIzbVPZy5z7sKp9lHFTX0KWfTBCtHaOk= github.com/ipfs/go-log/v2 v2.9.0/go.mod h1:UhIYAwMV7Nb4ZmihUxfIRM2Istw/y9cAk3xaK+4Zs2c= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= -github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= -github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= +github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= +github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.2 h1:TREegX1J4X+k1w4AhoDuxxFvVcS9SegMRvrmxF6Tca8= @@ -401,8 +401,8 @@ github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl9 github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= -github.com/libp2p/go-libp2p-kad-dht v0.36.0 h1:7QuXhV36+Vyj+L6A7mrYkn2sYLrbRcbjvsYDu/gXhn8= -github.com/libp2p/go-libp2p-kad-dht v0.36.0/go.mod h1:O24LxTH9Rt3I5XU8nmiA9VynS4TrTwAyj+zBJKB05vQ= +github.com/libp2p/go-libp2p-kad-dht v0.37.0 h1:V1IkFzK9taNS1UNAx260foulcBPH+watAUFjNo2qMUY= +github.com/libp2p/go-libp2p-kad-dht v0.37.0/go.mod h1:o4FPa1ea++UVAMJ1c+kyjUmj3CKm9+ZCyzQb4uutCFM= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= diff --git a/go.mod b/go.mod index b73a38d5a..daf5dd7a2 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd + github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 @@ -52,7 +52,7 @@ require ( github.com/libp2p/go-doh-resolver v0.5.0 github.com/libp2p/go-libp2p v0.46.0 github.com/libp2p/go-libp2p-http v0.5.0 - github.com/libp2p/go-libp2p-kad-dht v0.36.0 + github.com/libp2p/go-libp2p-kad-dht v0.37.0 github.com/libp2p/go-libp2p-kbucket v0.8.0 github.com/libp2p/go-libp2p-pubsub v0.15.0 github.com/libp2p/go-libp2p-pubsub-router v0.6.0 @@ -152,9 +152,9 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect - github.com/ipfs/go-ipfs-pq v0.0.3 // indirect + github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect - github.com/ipfs/go-peertaskqueue v0.8.2 // indirect + github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index ef94ed09b..ea56898d8 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -379,8 +379,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= -github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= -github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= +github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -400,8 +400,8 @@ github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6 github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-metrics-prometheus v0.1.0 h1:bApWOHkrH3VTBHzTHrZSfq4n4weOZDzZFxUXv+HyKcA= github.com/ipfs/go-metrics-prometheus v0.1.0/go.mod h1:2GtL525C/4yxtvSXpRJ4dnE45mCX9AS0XRa03vHx7G0= -github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= -github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= +github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= +github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.2 h1:TREegX1J4X+k1w4AhoDuxxFvVcS9SegMRvrmxF6Tca8= @@ -490,8 +490,8 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-http v0.5.0 h1:+x0AbLaUuLBArHubbbNRTsgWz0RjNTy6DJLOxQ3/QBc= github.com/libp2p/go-libp2p-http v0.5.0/go.mod h1:glh87nZ35XCQyFsdzZps6+F4HYI6DctVFY5u1fehwSg= -github.com/libp2p/go-libp2p-kad-dht v0.36.0 h1:7QuXhV36+Vyj+L6A7mrYkn2sYLrbRcbjvsYDu/gXhn8= -github.com/libp2p/go-libp2p-kad-dht v0.36.0/go.mod h1:O24LxTH9Rt3I5XU8nmiA9VynS4TrTwAyj+zBJKB05vQ= +github.com/libp2p/go-libp2p-kad-dht v0.37.0 h1:V1IkFzK9taNS1UNAx260foulcBPH+watAUFjNo2qMUY= +github.com/libp2p/go-libp2p-kad-dht v0.37.0/go.mod h1:o4FPa1ea++UVAMJ1c+kyjUmj3CKm9+ZCyzQb4uutCFM= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 83492ae30..d0d9fb364 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd // indirect + github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect @@ -183,7 +183,7 @@ require ( github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p v0.46.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.36.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.37.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index fbc1be308..a0cd4e152 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd h1:X7NB8jrFOSjM+19NH4JE6H7ApqtH/nBBiv2U6OIQxEA= -github.com/ipfs/boxo v0.35.3-0.20260120012647-ebdaf07cffbd/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= +github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= @@ -316,8 +316,8 @@ github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40 h1:M+5zwNetUg github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260119234300-242f1a10dc40/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= -github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= +github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= @@ -330,8 +330,8 @@ github.com/ipfs/go-log/v2 v2.9.0 h1:l4b06AwVXwldIzbVPZy5z7sKp9lHFTX0KWfTBCtHaOk= github.com/ipfs/go-log/v2 v2.9.0/go.mod h1:UhIYAwMV7Nb4ZmihUxfIRM2Istw/y9cAk3xaK+4Zs2c= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= -github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= -github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= +github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= +github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.2 h1:TREegX1J4X+k1w4AhoDuxxFvVcS9SegMRvrmxF6Tca8= @@ -419,8 +419,8 @@ github.com/libp2p/go-libp2p v0.46.0 h1:0T2yvIKpZ3DVYCuPOFxPD1layhRU486pj9rSlGWYn github.com/libp2p/go-libp2p v0.46.0/go.mod h1:TbIDnpDjBLa7isdgYpbxozIVPBTmM/7qKOJP4SFySrQ= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.36.0 h1:7QuXhV36+Vyj+L6A7mrYkn2sYLrbRcbjvsYDu/gXhn8= -github.com/libp2p/go-libp2p-kad-dht v0.36.0/go.mod h1:O24LxTH9Rt3I5XU8nmiA9VynS4TrTwAyj+zBJKB05vQ= +github.com/libp2p/go-libp2p-kad-dht v0.37.0 h1:V1IkFzK9taNS1UNAx260foulcBPH+watAUFjNo2qMUY= +github.com/libp2p/go-libp2p-kad-dht v0.37.0/go.mod h1:o4FPa1ea++UVAMJ1c+kyjUmj3CKm9+ZCyzQb4uutCFM= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= From 00fb95d69a878a6a742fcd7fb4fa8bbdac895251 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 22 Jan 2026 01:54:59 +0100 Subject: [PATCH 07/22] chore: apply suggestions from code review Co-authored-by: Andrew Gillis <11790789+gammazero@users.noreply.github.com> --- config/import.go | 2 +- test/cli/cid_profiles_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/import.go b/config/import.go index 995c371d2..2c88a12f3 100644 --- a/config/import.go +++ b/config/import.go @@ -31,7 +31,7 @@ const ( DefaultBatchMaxSize = 100 << 20 // 20MiB // HAMTSizeEstimation values for Import.UnixFSHAMTDirectorySizeEstimation - HAMTSizeEstimationLinks = "links" // legacy: estimate using link names + CID byte lengths + HAMTSizeEstimationLinks = "links" // legacy: estimate using link names + CID byte lengths (default) HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely diff --git a/test/cli/cid_profiles_test.go b/test/cli/cid_profiles_test.go index 4f153bf48..10e565283 100644 --- a/test/cli/cid_profiles_test.go +++ b/test/cli/cid_profiles_test.go @@ -350,7 +350,7 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, - "expected HAMT directory with <=%d links", exp.HAMTFanout) + "expected HAMT directory with <= %d links", exp.HAMTFanout) // Verify hash function verifyHashFunction(t, node, cidStr, exp.HashFunc) From caabe9e62b9688542df0b664cf13f528aca2d0a8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 22 Jan 2026 14:46:01 +0100 Subject: [PATCH 08/22] test(add): verify balanced DAG layout produces uniform leaf depth add test that confirms kubo uses balanced layout (all leaves at same depth) rather than balanced-packed (varying depths). creates 45MiB file to trigger multi-level DAG and walks it to verify leaf depth uniformity. includes trickle subtest to validate test logic can detect varying depths. supports CAR export via DAG_LAYOUT_CAR_OUTPUT env var for test vectors. --- test/cli/dag_layout_test.go | 147 ++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test/cli/dag_layout_test.go diff --git a/test/cli/dag_layout_test.go b/test/cli/dag_layout_test.go new file mode 100644 index 000000000..eb82a3387 --- /dev/null +++ b/test/cli/dag_layout_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/require" +) + +// TestBalancedDAGLayout verifies that kubo uses the "balanced" DAG layout +// (all leaves at same depth) rather than "balanced-packed" (varying leaf depths). +// +// DAG layout differences across implementations: +// +// - balanced: kubo, helia (all leaves at same depth, uniform traversal distance) +// - balanced-packed: singularity (trailing leaves may be at different depths) +// - trickle: kubo --trickle (varying depths, optimized for append-only/streaming) +// +// kubo does not implement balanced-packed. The trickle layout also produces +// non-uniform leaf depths but with different trade-offs: trickle is optimized +// for append-only and streaming reads (no seeking), while balanced-packed +// minimizes node count. +// +// IPIP-499 documents the balanced vs balanced-packed distinction. Files larger +// than dag_width ร— chunk_size will have different CIDs between implementations +// using different layouts. +// +// Set DAG_LAYOUT_CAR_OUTPUT environment variable to export CAR files. +// Example: DAG_LAYOUT_CAR_OUTPUT=/tmp/dag-layout go test -run TestBalancedDAGLayout -v +func TestBalancedDAGLayout(t *testing.T) { + t.Parallel() + + carOutputDir := os.Getenv("DAG_LAYOUT_CAR_OUTPUT") + exportCARs := carOutputDir != "" + if exportCARs { + if err := os.MkdirAll(carOutputDir, 0755); err != nil { + t.Fatalf("failed to create CAR output directory: %v", err) + } + t.Logf("CAR export enabled, writing to: %s", carOutputDir) + } + + t.Run("balanced layout has uniform leaf depth", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + + // Create file that triggers multi-level DAG. + // For default v0: 175 chunks ร— 256KiB = ~44.8 MiB (just over 174 max links) + // This creates a 2-level DAG where balanced layout ensures uniform depth. + fileSize := "45MiB" + seed := "balanced-test" + + cidStr := node.IPFSAddDeterministic(fileSize, seed) + + // Collect leaf depths by walking DAG + depths := collectLeafDepths(t, node, cidStr, 0) + + // All leaves must be at same depth for balanced layout + require.NotEmpty(t, depths, "expected at least one leaf node") + firstDepth := depths[0] + for i, d := range depths { + require.Equal(t, firstDepth, d, + "leaf %d at depth %d, expected %d (balanced layout requires uniform leaf depth)", + i, d, firstDepth) + } + t.Logf("verified %d leaves all at depth %d (CID: %s)", len(depths), firstDepth, cidStr) + + if exportCARs { + carPath := filepath.Join(carOutputDir, "balanced_"+fileSize+".car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + t.Run("trickle layout has varying leaf depth", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + + fileSize := "45MiB" + seed := "trickle-test" + + // Add with trickle layout (--trickle flag). + // Trickle produces non-uniform leaf depths, optimized for append-only + // and streaming reads (no seeking). This subtest validates the test + // logic by confirming we can detect varying depths. + cidStr := node.IPFSAddDeterministic(fileSize, seed, "--trickle") + + depths := collectLeafDepths(t, node, cidStr, 0) + + // Trickle layout should have varying depths + require.NotEmpty(t, depths, "expected at least one leaf node") + minDepth, maxDepth := depths[0], depths[0] + for _, d := range depths { + if d < minDepth { + minDepth = d + } + if d > maxDepth { + maxDepth = d + } + } + require.NotEqual(t, minDepth, maxDepth, + "trickle layout should have varying leaf depths, got uniform depth %d", minDepth) + t.Logf("verified %d leaves with depths ranging from %d to %d (CID: %s)", len(depths), minDepth, maxDepth, cidStr) + + if exportCARs { + carPath := filepath.Join(carOutputDir, "trickle_"+fileSize+".car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) +} + +// collectLeafDepths recursively walks DAG and returns depth of each leaf node. +// A node is a leaf if it's a raw block or a dag-pb node with no links. +func collectLeafDepths(t *testing.T, node *harness.Node, cid string, depth int) []int { + t.Helper() + + // Check codec to see if this is a raw leaf + res := node.IPFS("cid", "format", "-f", "%c", cid) + codec := strings.TrimSpace(res.Stdout.String()) + if codec == "raw" { + // Raw blocks are always leaves + return []int{depth} + } + + // Try to inspect as dag-pb node + pbNode, err := node.InspectPBNode(cid) + if err != nil { + // Can't parse as dag-pb, treat as leaf + return []int{depth} + } + + // No links = leaf node + if len(pbNode.Links) == 0 { + return []int{depth} + } + + // Recurse into children + var depths []int + for _, link := range pbNode.Links { + childDepths := collectLeafDepths(t, node, link.Hash.Slash, depth+1) + depths = append(depths, childDepths...) + } + return depths +} From 9b04a587f7da588dafa9948c85e54aee729fa007 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 27 Jan 2026 21:40:01 +0100 Subject: [PATCH 09/22] chore(deps): update boxo to 6141039ad8ef switches to https://github.com/ipfs/boxo/pull/1088/commits/6141039ad8ef098c3b65db8b2d1aeb3c16727c6c changes since 5cf22196ad0b: - refactor(unixfs): use arithmetic for exact block size calculation - refactor(unixfs): unify size tracking and make SizeEstimationMode immutable - feat(unixfs): optimize SizeEstimationBlock and add mode/mtime tests also clarifies that directory sharding globals affect both `ipfs add` and MFS. --- core/node/groups.go | 4 ++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/node/groups.go b/core/node/groups.go index ba6348db8..ab497e33b 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -438,10 +438,10 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { return fx.Error(err) } - // Auto-sharding settings + // Directory sharding settings from Import config. + // These globals affect both `ipfs add` and MFS (`ipfs files` API). shardSizeThreshold := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) shardMaxFanout := cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout) - // TODO: avoid overriding this globally, see if we can extend Directory interface like Get/SetMaxLinks from https://github.com/ipfs/boxo/pull/906 uio.HAMTShardingSize = int(shardSizeThreshold) uio.DefaultShardWidth = int(shardMaxFanout) uio.HAMTSizeEstimation = cfg.Import.HAMTSizeEstimationMode() diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 12908e7f0..4763a75db 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b + github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 73e6853eb..ce97affa8 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -265,8 +265,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef h1:yX6jGXVkt07Qb1u8rP0DgHMjPrzS/zo/VslLe+k3HwI= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index daf5dd7a2..5376d8e41 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b + github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index ea56898d8..5097d4c42 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef h1:yX6jGXVkt07Qb1u8rP0DgHMjPrzS/zo/VslLe+k3HwI= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index d0d9fb364..4af654ca4 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b // indirect + github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index a0cd4e152..beb8a32ef 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b h1:lV0MHFwdyAXnSEP8AUOYNaY97+DJAjgaHPMKVGFBorw= -github.com/ipfs/boxo v0.35.3-0.20260121071626-5cf22196ad0b/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef h1:yX6jGXVkt07Qb1u8rP0DgHMjPrzS/zo/VslLe+k3HwI= +github.com/ipfs/boxo v0.35.3-0.20260127202919-6141039ad8ef/go.mod h1:NpWvKO86jiZaucHQXf9VWqWRzI5qPbrPuAZCm4AnSas= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 120800fe8b499b075ffc2c567b4d4aeae69cd7e7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 27 Jan 2026 22:38:24 +0100 Subject: [PATCH 10/22] test(cli): improve HAMT threshold tests with exact +1 byte verification - add UnixFSDataType() helper to directly check UnixFS type via protobuf - refactor threshold tests to use exact +1 byte calculations instead of +1 file - verify directory type directly (ft.TDirectory vs ft.THAMTShard) instead of inferring from link count - clean up helper function signatures by removing unused cidLength parameter --- test/cli/add_test.go | 108 +++++++++++++++------------------- test/cli/cid_profiles_test.go | 24 ++++++-- test/cli/harness/pbinspect.go | 40 ++++++++++++- 3 files changed, 105 insertions(+), 67 deletions(-) diff --git a/test/cli/add_test.go b/test/cli/add_test.go index 03abb821d..68cfda7e2 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" @@ -251,21 +252,20 @@ func TestAdd(t *testing.T) { // Create directory exactly at the 256KiB threshold using links estimation. // Links estimation: size = numFiles * (nameLen + cidLen) // 4096 * (30 + 34) = 4096 * 64 = 262144 = threshold exactly - // With > comparison: stays as basic directory - // With >= comparison: converts to HAMT + // Threshold uses > not >=, so directory at exact threshold stays basic. const numFiles, nameLen = 4096, 30 - err = createDirectoryForHAMTLinksEstimation(randDir, cidV0Length, numFiles, nameLen, nameLen, seed) + err = createDirectoryForHAMTLinksEstimation(randDir, numFiles, nameLen, nameLen, seed) require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Should remain a basic directory (threshold uses > not >=) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, numFiles, len(root.Links), "expected basic directory at exact threshold") + // Verify it's a basic directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") }) - t.Run("over UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { + t.Run("1 byte over UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -274,19 +274,19 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory just over the 256KiB threshold using links estimation. - // Links estimation: size = numFiles * (nameLen + cidLen) - // 4097 * (30 + 34) = 4097 * 64 = 262208 > 262144, exceeds threshold - const numFiles, nameLen = 4097, 30 - err = createDirectoryForHAMTLinksEstimation(randDir, cidV0Length, numFiles, nameLen, nameLen, seed) + // Create directory exactly 1 byte over the 256KiB threshold using links estimation. + // Links estimation: size = sum(nameLen + cidLen) for each file + // 4095 * (30 + 34) + 1 * (31 + 34) = 262080 + 65 = 262145 = threshold + 1 + const numFiles, nameLen, lastNameLen = 4096, 30, 31 + err = createDirectoryForHAMTLinksEstimation(randDir, numFiles, nameLen, lastNameLen, seed) require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Should be HAMT sharded (root links <= fanout of 256) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.LessOrEqual(t, len(root.Links), 256, "expected HAMT directory when over threshold") + // Verify it's a HAMT directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when 1 byte over threshold") }) }) @@ -349,24 +349,23 @@ func TestAdd(t *testing.T) { require.NoError(t, err) // Create directory exactly at the 256KiB threshold using block estimation. - // Block estimation: size = baseOverhead + numFiles * LinkSerializedSize - // LinkSerializedSize(11, 36, 0) = 55 bytes per link - // 4766 * 55 + 14 = 262130 + 14 = 262144 = threshold exactly - // With > comparison: stays as basic directory - // With >= comparison: converts to HAMT - const numFiles, nameLen = 4766, 11 - err = createDirectoryForHAMTBlockEstimation(randDir, cidV1Length, numFiles, nameLen, nameLen, seed) + // Block estimation: size = dataFieldSize + sum(LinkSerializedSize) + // - dataFieldSerializedSize() = 4 bytes (empty dir, no mode/mtime) + // - LinkSerializedSize(nameLen=11) = 55 bytes, LinkSerializedSize(nameLen=21) = 65 bytes + // Total: 4765 * 55 + 65 + 4 = 262144 = threshold exactly + const numFiles, nameLen, lastNameLen = 4766, 11, 21 + err = createDirectoryForHAMTBlockEstimation(randDir, numFiles, nameLen, lastNameLen, seed) require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Should remain a basic directory (threshold uses > not >=) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, numFiles, len(root.Links), "expected basic directory at exact threshold") + // Verify it's a basic directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") }) - t.Run("over UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { + t.Run("1 byte over UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(profile) node.StartDaemon() @@ -375,19 +374,18 @@ func TestAdd(t *testing.T) { randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create directory just over the 256KiB threshold using block estimation. - // Block estimation: size = baseOverhead + numFiles * LinkSerializedSize - // 4767 * 55 + 14 = 262185 + 14 = 262199 > 262144, exceeds threshold - const numFiles, nameLen = 4767, 11 - err = createDirectoryForHAMTBlockEstimation(randDir, cidV1Length, numFiles, nameLen, nameLen, seed) + // Create directory exactly 1 byte over the 256KiB threshold. + // Same as above but lastNameLen=22 adds 1 byte: 4765 * 55 + 66 + 4 = 262145 + const numFiles, nameLen, lastNameLen = 4766, 11, 22 + err = createDirectoryForHAMTBlockEstimation(randDir, numFiles, nameLen, lastNameLen, seed) require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Should be HAMT sharded (root links <= fanout of 256) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.LessOrEqual(t, len(root.Links), 256, "expected HAMT directory when over threshold") + // Verify it's a HAMT directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when 1 byte over threshold") }) }) @@ -398,8 +396,8 @@ func TestAdd(t *testing.T) { setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "hidden-test") require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0o644)) return testDir } @@ -447,8 +445,8 @@ func TestAdd(t *testing.T) { setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "empty-dirs-test") require.NoError(t, err) - require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0644)) + require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0o644)) return testDir } @@ -502,14 +500,14 @@ func TestAdd(t *testing.T) { // Top-level file and symlink targetFile := filepath.Join(testDir, "target.txt") - require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0644)) + require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0o644)) require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt"))) // Nested file and symlink in sub-sub directory subsubdir := filepath.Join(testDir, "subdir", "subsubdir") - require.NoError(t, os.MkdirAll(subsubdir, 0755)) + require.NoError(t, os.MkdirAll(subsubdir, 0o755)) nestedTarget := filepath.Join(subsubdir, "nested-target.txt") - require.NoError(t, os.WriteFile(nestedTarget, []byte("nested content"), 0644)) + require.NoError(t, os.WriteFile(nestedTarget, []byte("nested content"), 0o644)) require.NoError(t, os.Symlink("nested-target.txt", filepath.Join(subsubdir, "nested-link.txt"))) return testDir @@ -806,32 +804,22 @@ func TestAddFastProvide(t *testing.T) { } // createDirectoryForHAMTLinksEstimation creates a directory with the specified number -// of files using the links-based size estimation formula (size = numFiles * (nameLen + cidLen)). +// of files for testing links-based size estimation (size = sum of nameLen + cidLen). // Used by legacy profiles (unixfs-v0-2015). // -// Threshold behavior: boxo uses > comparison, so directory at exact threshold stays basic. -// Use DirBasicFiles for basic directory test, DirHAMTFiles for HAMT directory test. -// // The lastNameLen parameter allows the last file to have a different name length, // enabling exact +1 byte threshold tests. -// -// See boxo/ipld/unixfs/io/directory.go sizeBelowThreshold() for the links estimation. -func createDirectoryForHAMTLinksEstimation(dirPath string, cidLength, numFiles, nameLen, lastNameLen int, seed string) error { +func createDirectoryForHAMTLinksEstimation(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) } // createDirectoryForHAMTBlockEstimation creates a directory with the specified number -// of files using the block-based size estimation formula (LinkSerializedSize with protobuf overhead). +// of files for testing block-based size estimation (LinkSerializedSize with protobuf overhead). // Used by modern profiles (unixfs-v1-2025). // -// Threshold behavior: boxo uses > comparison, so directory at exact threshold stays basic. -// Use DirBasicFiles for basic directory test, DirHAMTFiles for HAMT directory test. -// // The lastNameLen parameter allows the last file to have a different name length, // enabling exact +1 byte threshold tests. -// -// See boxo/ipld/unixfs/io/directory.go estimatedBlockSize() for the block estimation. -func createDirectoryForHAMTBlockEstimation(dirPath string, cidLength, numFiles, nameLen, lastNameLen int, seed string) error { +func createDirectoryForHAMTBlockEstimation(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) } @@ -870,7 +858,7 @@ func createDeterministicFiles(dirPath string, numFiles, nameLen, lastNameLen int filePath := filepath.Join(dirPath, filename) // Create file with 1-byte content for non-zero tsize - if err := os.WriteFile(filePath, []byte("x"), 0644); err != nil { + if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { return err } } diff --git a/test/cli/cid_profiles_test.go b/test/cli/cid_profiles_test.go index 10e565283..76021c79d 100644 --- a/test/cli/cid_profiles_test.go +++ b/test/cli/cid_profiles_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" @@ -133,7 +134,7 @@ func TestCIDProfiles(t *testing.T) { carOutputDir := os.Getenv("CID_PROFILES_CAR_OUTPUT") exportCARs := carOutputDir != "" if exportCARs { - if err := os.MkdirAll(carOutputDir, 0755); err != nil { + if err := os.MkdirAll(carOutputDir, 0o755); err != nil { t.Fatalf("failed to create CAR output directory: %v", err) } t.Logf("CAR export enabled, writing to: %s", carOutputDir) @@ -276,14 +277,19 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri basicLastNameLen = exp.DirBasicNameLen } if exp.HAMTSizeEstimation == "block" { - err = createDirectoryForHAMTBlockEstimation(randDir, cidLen, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) + err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) } else { - err = createDirectoryForHAMTLinksEstimation(randDir, cidLen, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) + err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) } require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + // Verify it's a basic directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") + root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, exp.DirBasicFiles, len(root.Links), @@ -339,18 +345,24 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri lastNameLen = exp.DirHAMTNameLen } if exp.HAMTSizeEstimation == "block" { - err = createDirectoryForHAMTBlockEstimation(randDir, cidLen, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) + err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) } else { - err = createDirectoryForHAMTLinksEstimation(randDir, cidLen, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) + err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) } require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + // Verify it's a HAMT directory by checking UnixFS type + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when over threshold") + + // HAMT root has at most fanout links (actual count depends on hash distribution) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, - "expected HAMT directory with <= %d links", exp.HAMTFanout) + "expected HAMT directory root to have <= %d links", exp.HAMTFanout) // Verify hash function verifyHashFunction(t, node, cidStr, exp.HashFunc) diff --git a/test/cli/harness/pbinspect.go b/test/cli/harness/pbinspect.go index 6abddb61f..6210e7bed 100644 --- a/test/cli/harness/pbinspect.go +++ b/test/cli/harness/pbinspect.go @@ -3,8 +3,47 @@ package harness import ( "bytes" "encoding/json" + + mdag "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" + pb "github.com/ipfs/boxo/ipld/unixfs/pb" ) +// UnixFSDataType returns the UnixFS DataType for the given CID by fetching the +// raw block and parsing the protobuf. This directly checks the Type field in +// the UnixFS Data message (https://specs.ipfs.tech/unixfs/#data). +// +// Common types: +// - ft.TDirectory (1) = basic flat directory +// - ft.THAMTShard (5) = HAMT sharded directory +func (n *Node) UnixFSDataType(cid string) (pb.Data_DataType, error) { + log.Debugf("node %d block get %s", n.ID, cid) + + var blockData bytes.Buffer + res := n.Runner.MustRun(RunRequest{ + Path: n.IPFSBin, + Args: []string{"block", "get", cid}, + CmdOpts: []CmdOpt{RunWithStdout(&blockData)}, + }) + if res.Err != nil { + return 0, res.Err + } + + // Parse dag-pb block + protoNode, err := mdag.DecodeProtobuf(blockData.Bytes()) + if err != nil { + return 0, err + } + + // Parse UnixFS data + fsNode, err := ft.FSNodeFromBytes(protoNode.Data()) + if err != nil { + return 0, err + } + + return fsNode.Type(), nil +} + // InspectPBNode uses dag-json output of 'ipfs dag get' to inspect // "Logical Format" of DAG-PB as defined in // https://web.archive.org/web/20250403194752/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format @@ -28,7 +67,6 @@ func (n *Node) InspectPBNode(cid string) (PBNode, error) { return root, err } return root, nil - } // Define structs to match the JSON for From 9500a5289b5d8eed8c8e767a7297868cc00cd2a4 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 27 Jan 2026 23:35:14 +0100 Subject: [PATCH 11/22] test(cli): consolidate profile tests into cid_profiles_test.go remove duplicate profile threshold tests from add_test.go since they are fully covered by the data-driven tests in cid_profiles_test.go. changes: - improve test names to describe what threshold is being tested - add inline documentation explaining each test's purpose - add byte-precise helper IPFSAddDeterministicBytes for threshold tests - remove ~200 lines of duplicated test code from add_test.go - keep non-profile tests (pinning, symlinks, hidden files) in add_test.go --- test/cli/add_test.go | 209 +------------- test/cli/cid_profiles_test.go | 318 ++++++++++++++------- test/cli/harness/ipfs.go | 14 +- test/cli/testutils/random_deterministic.go | 12 +- 4 files changed, 243 insertions(+), 310 deletions(-) diff --git a/test/cli/add_test.go b/test/cli/add_test.go index 68cfda7e2..323aab5c7 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" @@ -40,11 +39,6 @@ func TestAdd(t *testing.T) { shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6" ) - const ( - cidV0Length = 34 // cidv0 sha2-256 - cidV1Length = 36 // cidv1 sha2-256 - ) - t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() @@ -187,207 +181,8 @@ func TestAdd(t *testing.T) { require.Equal(t, "QmbBftNHWmjSWKLC49dMVrfnY8pjrJYntiAXirFJ7oJrNk", cidStr) }) - t.Run("ipfs init --profile=unixfs-v0-2015 sets config that produces legacy CIDv0", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init("--profile=unixfs-v0-2015") - node.StartDaemon() - defer node.StopDaemon() - - cidStr := node.IPFSAddStr(shortString) - require.Equal(t, shortStringCidV0, cidStr) - }) - - t.Run("ipfs init --profile=unixfs-v0-2015 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks", func(t *testing.T) { - t.Parallel() - seed := "v0-seed" - profile := "--profile=unixfs-v0-2015" - - t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // Add 44544KiB file: - // 174 * 256KiB should fit in single DAG layer - cidStr := node.IPFSAddDeterministic("44544KiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 174, len(root.Links)) - // expect same CID every time - require.Equal(t, "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", cidStr) - }) - - t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // add 256KiB (one more block), it should force rebalancing DAG and moving most to second layer - cidStr := node.IPFSAddDeterministic("44800KiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 2, len(root.Links)) - // expect same CID every time - require.Equal(t, "QmepeWtdmS1hHXx1oZXsPUv6bMrfRRKfZcoPPU4eEfjnbf", cidStr) - }) - }) - - t.Run("ipfs init --profile=unixfs-v0-2015 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { - t.Parallel() - seed := "hamt-unixfs-v0-2015" - profile := "--profile=unixfs-v0-2015" - - // unixfs-v0-2015 uses links-based estimation: size = sum(nameLen + cidLen) - // Threshold is 256KiB = 262144 bytes - - t.Run("at UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory exactly at the 256KiB threshold using links estimation. - // Links estimation: size = numFiles * (nameLen + cidLen) - // 4096 * (30 + 34) = 4096 * 64 = 262144 = threshold exactly - // Threshold uses > not >=, so directory at exact threshold stays basic. - const numFiles, nameLen = 4096, 30 - err = createDirectoryForHAMTLinksEstimation(randDir, numFiles, nameLen, nameLen, seed) - require.NoError(t, err) - - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Verify it's a basic directory by checking UnixFS type - fsType, err := node.UnixFSDataType(cidStr) - require.NoError(t, err) - require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") - }) - - t.Run("1 byte over UnixFSHAMTDirectorySizeThreshold=256KiB (links estimation)", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory exactly 1 byte over the 256KiB threshold using links estimation. - // Links estimation: size = sum(nameLen + cidLen) for each file - // 4095 * (30 + 34) + 1 * (31 + 34) = 262080 + 65 = 262145 = threshold + 1 - const numFiles, nameLen, lastNameLen = 4096, 30, 31 - err = createDirectoryForHAMTLinksEstimation(randDir, numFiles, nameLen, lastNameLen, seed) - require.NoError(t, err) - - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Verify it's a HAMT directory by checking UnixFS type - fsType, err := node.UnixFSDataType(cidStr) - require.NoError(t, err) - require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when 1 byte over threshold") - }) - }) - - t.Run("ipfs init --profile=unixfs-v1-2025 produces CIDv1 with raw leaves", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init("--profile=unixfs-v1-2025") - node.StartDaemon() - defer node.StopDaemon() - - cidStr := node.IPFSAddStr(shortString) - require.Equal(t, shortStringCidV1, cidStr) // raw leaf - }) - - t.Run("ipfs init --profile=unixfs-v1-2025 applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=1024", func(t *testing.T) { - t.Parallel() - seed := "v1-2025-seed" - profile := "--profile=unixfs-v1-2025" - - t.Run("under UnixFSFileMaxLinks=1024", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // 1024 * 1MiB should fit in single layer - cidStr := node.IPFSAddDeterministic("1024MiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 1024, len(root.Links)) - }) - - t.Run("above UnixFSFileMaxLinks=1024", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - // add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer - cidStr := node.IPFSAddDeterministic("1025MiB", seed) - root, err := node.InspectPBNode(cidStr) - assert.NoError(t, err) - require.Equal(t, 2, len(root.Links)) - }) - }) - - t.Run("ipfs init --profile=unixfs-v1-2025 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) { - t.Parallel() - seed := "hamt-unixfs-v1-2025" - profile := "--profile=unixfs-v1-2025" - - // unixfs-v1-2025 uses block-based size estimation: size = sum(LinkSerializedSize) - // where LinkSerializedSize includes protobuf overhead (tags, varints, wrappers). - // Threshold is 256KiB = 262144 bytes - - t.Run("at UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory exactly at the 256KiB threshold using block estimation. - // Block estimation: size = dataFieldSize + sum(LinkSerializedSize) - // - dataFieldSerializedSize() = 4 bytes (empty dir, no mode/mtime) - // - LinkSerializedSize(nameLen=11) = 55 bytes, LinkSerializedSize(nameLen=21) = 65 bytes - // Total: 4765 * 55 + 65 + 4 = 262144 = threshold exactly - const numFiles, nameLen, lastNameLen = 4766, 11, 21 - err = createDirectoryForHAMTBlockEstimation(randDir, numFiles, nameLen, lastNameLen, seed) - require.NoError(t, err) - - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Verify it's a basic directory by checking UnixFS type - fsType, err := node.UnixFSDataType(cidStr) - require.NoError(t, err) - require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") - }) - - t.Run("1 byte over UnixFSHAMTDirectorySizeThreshold=256KiB (block estimation)", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init(profile) - node.StartDaemon() - defer node.StopDaemon() - - randDir, err := os.MkdirTemp(node.Dir, seed) - require.NoError(t, err) - - // Create directory exactly 1 byte over the 256KiB threshold. - // Same as above but lastNameLen=22 adds 1 byte: 4765 * 55 + 66 + 4 = 262145 - const numFiles, nameLen, lastNameLen = 4766, 11, 22 - err = createDirectoryForHAMTBlockEstimation(randDir, numFiles, nameLen, lastNameLen, seed) - require.NoError(t, err) - - cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - - // Verify it's a HAMT directory by checking UnixFS type - fsType, err := node.UnixFSDataType(cidStr) - require.NoError(t, err) - require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when 1 byte over threshold") - }) - }) + // Profile-specific threshold tests are in cid_profiles_test.go (TestCIDProfiles). + // Tests here cover general ipfs add behavior not tied to specific profiles. t.Run("ipfs add --hidden", func(t *testing.T) { t.Parallel() diff --git a/test/cli/cid_profiles_test.go b/test/cli/cid_profiles_test.go index 76021c79d..65abc747b 100644 --- a/test/cli/cid_profiles_test.go +++ b/test/cli/cid_profiles_test.go @@ -16,6 +16,12 @@ import ( // cidProfileExpectations defines expected behaviors for a UnixFS import profile. // This allows DRY testing of multiple profiles with the same test logic. +// +// Each profile is tested against threshold boundaries to verify: +// - CID format (version, hash function, raw leaves vs dag-pb wrapped) +// - File chunking (UnixFSChunker size threshold) +// - DAG structure (UnixFSFileMaxLinks rebalancing threshold) +// - Directory sharding (HAMTThreshold for flat vs HAMT directories) type cidProfileExpectations struct { // Profile identification Name string // canonical profile name from IPIP-499 @@ -26,11 +32,12 @@ type cidProfileExpectations struct { HashFunc string // e.g., "sha2-256" RawLeaves bool // true = raw codec for small files, false = dag-pb wrapped - // File chunking expectations - ChunkSize string // e.g., "1MiB" or "256KiB" - FileMaxLinks int // max links before DAG rebalancing + // File chunking expectations (UnixFSChunker config) + ChunkSize int // chunk size in bytes (e.g., 262144 for 256KiB, 1048576 for 1MiB) + ChunkSizeHuman string // human-readable chunk size (e.g., "256KiB", "1MiB") + FileMaxLinks int // max links before DAG rebalancing (UnixFSFileMaxLinks config) - // HAMT directory sharding expectations. + // HAMT directory sharding expectations (UnixFSHAMTDirectory* config). // Threshold behavior: boxo converts to HAMT when size > HAMTThreshold (not >=). // This means a directory exactly at the threshold stays as a basic (flat) directory. HAMTFanout int // max links per HAMT shard bucket (256) @@ -48,12 +55,40 @@ type cidProfileExpectations struct { DirHAMTLastNameLen int // filename length for last file (0 = same as DirHAMTNameLen) DirHAMTFiles int // total file count for HAMT directory (over threshold) - // Expected deterministic CIDs for test vectors - SmallFileCID string // CID for single byte "x" - FileAtMaxLinksCID string // CID for file at max links - FileOverMaxLinksCID string // CID for file triggering rebalance - DirBasicCID string // CID for basic directory (at exact threshold, stays flat) - DirHAMTCID string // CID for HAMT directory (over threshold, sharded) + // Expected deterministic CIDs for test vectors. + // These serve as regression tests to detect unintended changes in CID generation. + + // SmallFileCID is the deterministic CID for "hello world" string. + // Tests basic CID format (version, codec, hash). + SmallFileCID string + + // FileAtChunkSizeCID is the deterministic CID for a file exactly at chunk size. + // This file fits in a single block with no links: + // - v0-2015: dag-pb wrapped TFile node (CIDv0) + // - v1-2025: raw leaf block (CIDv1) + FileAtChunkSizeCID string + + // FileOverChunkSizeCID is the deterministic CID for a file 1 byte over chunk size. + // This file requires 2 chunks, producing a root dag-pb node with 2 links: + // - v0-2015: links point to dag-pb wrapped TFile leaf nodes + // - v1-2025: links point to raw leaf blocks + FileOverChunkSizeCID string + + // FileAtMaxLinksCID is the deterministic CID for a file at UnixFSFileMaxLinks threshold. + // File size = maxLinks * chunkSize, producing a single-layer DAG with exactly maxLinks children. + FileAtMaxLinksCID string + + // FileOverMaxLinksCID is the deterministic CID for a file 1 byte over max links threshold. + // The +1 byte requires an additional chunk, forcing DAG rebalancing to 2 layers. + FileOverMaxLinksCID string + + // DirBasicCID is the deterministic CID for a directory exactly at HAMTThreshold. + // With > comparison (not >=), directory at exact threshold stays as basic (flat) directory. + DirBasicCID string + + // DirHAMTCID is the deterministic CID for a directory 1 byte over HAMTThreshold. + // Crossing the threshold converts the directory to a HAMT sharded structure. + DirHAMTCID string } // unixfsV02015 is the legacy profile for backward-compatible CID generation. @@ -66,8 +101,9 @@ var unixfsV02015 = cidProfileExpectations{ HashFunc: "sha2-256", RawLeaves: false, - ChunkSize: "256KiB", - FileMaxLinks: 174, + ChunkSize: 262144, // 256 KiB + ChunkSizeHuman: "256KiB", + FileMaxLinks: 174, HAMTFanout: 256, HAMTThreshold: 262144, // 256 KiB @@ -78,11 +114,13 @@ var unixfsV02015 = cidProfileExpectations{ DirHAMTLastNameLen: 0, // 0 = same as DirHAMTNameLen (uniform filenames) DirHAMTFiles: 4033, // 4033 * 65 = 262145 (becomes HAMT) - SmallFileCID: "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", // "hello world" dag-pb wrapped - FileAtMaxLinksCID: "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", // 44544KiB with seed "v0-seed" - FileOverMaxLinksCID: "QmepeWtdmS1hHXx1oZXsPUv6bMrfRRKfZcoPPU4eEfjnbf", // 44800KiB with seed "v0-seed" - DirBasicCID: "QmX5GtRk3TSSEHtdrykgqm4eqMEn3n2XhfkFAis5fjyZmN", // 4096 files at threshold - DirHAMTCID: "QmeMiJzmhpJAUgynAcxTQYek5PPKgdv3qEvFsdV3XpVnvP", // 4033 files +1 over threshold + SmallFileCID: "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", // "hello world" dag-pb wrapped + FileAtChunkSizeCID: "QmWmRj3dFDZdb6ABvbmKhEL6TmPbAfBZ1t5BxsEyJrcZhE", // 262144 bytes with seed "chunk-v0-seed" + FileOverChunkSizeCID: "QmYyLxtzZyW22zpoVAtKANLRHpDjZtNeDjQdJrcQNWoRkJ", // 262145 bytes with seed "chunk-v0-seed" + FileAtMaxLinksCID: "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", // 174*256KiB bytes with seed "v0-seed" + FileOverMaxLinksCID: "QmV81WL765sC8DXsRhE5fJv2rwhS4icHRaf3J9Zk5FdRnW", // 174*256KiB+1 bytes with seed "v0-seed" + DirBasicCID: "QmX5GtRk3TSSEHtdrykgqm4eqMEn3n2XhfkFAis5fjyZmN", // 4096 files at threshold + DirHAMTCID: "QmeMiJzmhpJAUgynAcxTQYek5PPKgdv3qEvFsdV3XpVnvP", // 4033 files +1 over threshold } // unixfsV12025 is the recommended profile for cross-implementation CID determinism. @@ -94,8 +132,9 @@ var unixfsV12025 = cidProfileExpectations{ HashFunc: "sha2-256", RawLeaves: true, - ChunkSize: "1MiB", - FileMaxLinks: 1024, + ChunkSize: 1048576, // 1 MiB + ChunkSizeHuman: "1MiB", + FileMaxLinks: 1024, HAMTFanout: 256, HAMTThreshold: 262144, // 256 KiB @@ -109,11 +148,13 @@ var unixfsV12025 = cidProfileExpectations{ DirHAMTLastNameLen: 22, // last file: 66 bytes; total: 4765*55 + 66 + 4 = 262145 (+1 over threshold) DirHAMTFiles: 4766, // becomes HAMT - SmallFileCID: "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", // "hello world" raw leaf - FileAtMaxLinksCID: "bafybeihmf37wcuvtx4hpu7he5zl5qaf2ineo2lqlfrapokkm5zzw7zyhvm", // 1024MiB with seed "v1-2025-seed" - FileOverMaxLinksCID: "bafybeihmzokxxjqwxjcryerhp5ezpcog2wcawfryb2xm64xiakgm4a5jue", // 1025MiB with seed "v1-2025-seed" - DirBasicCID: "bafybeic3h7rwruealwxkacabdy45jivq2crwz6bufb5ljwupn36gicplx4", // 4766 files at 262144 bytes (threshold) - DirHAMTCID: "bafybeiegvuterwurhdtkikfhbxcldohmxp566vpjdofhzmnhv6o4freidu", // 4766 files at 262145 bytes (+1 over) + SmallFileCID: "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", // "hello world" raw leaf + FileAtChunkSizeCID: "bafkreiacndfy443ter6qr2tmbbdhadvxxheowwf75s6zehscklu6ezxmta", // 1048576 bytes with seed "chunk-v1-seed" + FileOverChunkSizeCID: "bafybeigmix7t42i6jacydtquhet7srwvgpizfg7gjbq7627d35mjomtu64", // 1048577 bytes with seed "chunk-v1-seed" + FileAtMaxLinksCID: "bafybeihmf37wcuvtx4hpu7he5zl5qaf2ineo2lqlfrapokkm5zzw7zyhvm", // 1024*1MiB bytes with seed "v1-2025-seed" + FileOverMaxLinksCID: "bafybeibdsi225ugbkmpbdohnxioyab6jsqrmkts3twhpvfnzp77xtzpyhe", // 1024*1MiB+1 bytes with seed "v1-2025-seed" + DirBasicCID: "bafybeic3h7rwruealwxkacabdy45jivq2crwz6bufb5ljwupn36gicplx4", // 4766 files at 262144 bytes (threshold) + DirHAMTCID: "bafybeiegvuterwurhdtkikfhbxcldohmxp566vpjdofhzmnhv6o4freidu", // 4766 files at 262145 bytes (+1 over) } // defaultProfile points to the profile that matches Kubo's implicit default behavior. @@ -160,31 +201,40 @@ func TestCIDProfiles(t *testing.T) { } // runProfileTests runs all test vectors for a given profile. +// Tests verify threshold behaviors for: +// - Small files (CID format verification) +// - UnixFSChunker threshold (single block vs multi-block) +// - UnixFSFileMaxLinks threshold (single-layer vs rebalanced DAG) +// - HAMTThreshold (basic flat directory vs HAMT sharded) func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir string, exportCARs bool) { cidLen := cidV0Length if exp.CIDVersion == 1 { cidLen = cidV1Length } - t.Run("small-file", func(t *testing.T) { + // Test: small file produces correct CID format + // Verifies the profile sets the expected CID version, hash function, and leaf encoding. + t.Run("small file produces correct CID format", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() - // Use "hello world" for determinism - matches CIDs in add_test.go + // Use "hello world" for determinism cidStr := node.IPFSAddStr("hello world") - // Verify CID version + // Verify CID version (v0 starts with "Qm", v1 with "b") verifyCIDVersion(t, node, cidStr, exp.CIDVersion) - // Verify hash function + // Verify hash function (sha2-256 for both profiles) verifyHashFunction(t, node, cidStr, exp.HashFunc) - // Verify raw leaves vs wrapped + // Verify raw leaves vs dag-pb wrapped + // - v0-2015: dag-pb codec (wrapped) + // - v1-2025: raw codec (raw leaves) verifyRawLeaves(t, node, cidStr, exp.RawLeaves) - // Verify deterministic CID if expected + // Verify deterministic CID matches expected value if exp.SmallFileCID != "" { require.Equal(t, exp.SmallFileCID, cidStr, "expected deterministic CID for small file") } @@ -196,27 +246,116 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri } }) - t.Run("file-at-max-links", func(t *testing.T) { + // Test: file at UnixFSChunker threshold (single block) + // A file exactly at chunk size fits in one block with no links. + // - v0-2015 (256KiB): produces dag-pb wrapped TFile node + // - v1-2025 (1MiB): produces raw leaf block + t.Run("file at UnixFSChunker threshold (single block)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() - // Calculate file size: maxLinks * chunkSize - fileSize := fileAtMaxLinksSize(exp) - // Seed matches add_test.go for deterministic CIDs + // File exactly at chunk size = single block (no links) + seed := chunkSeedForProfile(exp) + cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize), seed) + + // Verify block structure based on raw leaves setting + if exp.RawLeaves { + // v1-2025: single block is a raw leaf (no dag-pb structure) + codec := node.IPFS("cid", "format", "-f", "%c", cidStr).Stdout.Trimmed() + require.Equal(t, "raw", codec, "single block file is raw leaf") + } else { + // v0-2015: single block is a dag-pb node with no links (TFile type) + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.Equal(t, 0, len(root.Links), "single block file has no links") + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TFile, fsType, "single block file is dag-pb wrapped (TFile)") + } + + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + if exp.FileAtChunkSizeCID != "" { + require.Equal(t, exp.FileAtChunkSizeCID, cidStr, "expected deterministic CID for file at chunk size") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-chunk-size.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + // Test: file 1 byte over UnixFSChunker threshold (2 blocks) + // A file 1 byte over chunk size requires 2 chunks. + // Root is a dag-pb node with 2 links. Leaf encoding depends on profile: + // - v0-2015: leaf blocks are dag-pb wrapped TFile nodes + // - v1-2025: leaf blocks are raw codec blocks + t.Run("file 1 byte over UnixFSChunker threshold (2 blocks)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // File +1 byte over chunk size = 2 blocks + seed := chunkSeedForProfile(exp) + cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize)+1, seed) + + root, err := node.InspectPBNode(cidStr) + assert.NoError(t, err) + require.Equal(t, 2, len(root.Links), "file over chunk size has 2 links") + + // Verify leaf block encoding + for _, link := range root.Links { + if exp.RawLeaves { + // v1-2025: leaves are raw blocks + leafCodec := node.IPFS("cid", "format", "-f", "%c", link.Hash.Slash).Stdout.Trimmed() + require.Equal(t, "raw", leafCodec, "leaf blocks are raw, not dag-pb") + } else { + // v0-2015: leaves are dag-pb wrapped (TFile type) + leafType, err := node.UnixFSDataType(link.Hash.Slash) + require.NoError(t, err) + require.Equal(t, ft.TFile, leafType, "leaf blocks are dag-pb wrapped (TFile)") + } + } + + verifyHashFunction(t, node, cidStr, exp.HashFunc) + + if exp.FileOverChunkSizeCID != "" { + require.Equal(t, exp.FileOverChunkSizeCID, cidStr, "expected deterministic CID for file over chunk size") + } + + if exportCARs { + carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-chunk-size.car") + require.NoError(t, node.IPFSDagExport(cidStr, carPath)) + t.Logf("exported: %s -> %s", cidStr, carPath) + } + }) + + // Test: file at UnixFSFileMaxLinks threshold (single layer) + // A file of exactly maxLinks * chunkSize bytes fits in a single DAG layer. + // - v0-2015: 174 links (174 * 256KiB = ~44.6MiB) + // - v1-2025: 1024 links (1024 * 1MiB = 1GiB) + t.Run("file at UnixFSFileMaxLinks threshold (single layer)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) + node.StartDaemon() + defer node.StopDaemon() + + // File size = maxLinks * chunkSize (exactly at threshold) + fileSize := fileAtMaxLinksBytes(exp) seed := seedForProfile(exp) - cidStr := node.IPFSAddDeterministic(fileSize, seed) + cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, exp.FileMaxLinks, len(root.Links), "expected exactly %d links at max", exp.FileMaxLinks) - // Verify hash function on root verifyHashFunction(t, node, cidStr, exp.HashFunc) - // Verify deterministic CID if expected if exp.FileAtMaxLinksCID != "" { require.Equal(t, exp.FileAtMaxLinksCID, cidStr, "expected deterministic CID for file at max links") } @@ -228,26 +367,27 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri } }) - t.Run("file-over-max-links-rebalanced", func(t *testing.T) { + // Test: file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG) + // Adding 1 byte requires an additional chunk, exceeding maxLinks. + // This triggers DAG rebalancing: chunks are grouped into intermediate nodes, + // producing a 2-layer DAG with 2 links at the root. + t.Run("file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() - // One more chunk triggers rebalancing - fileSize := fileOverMaxLinksSize(exp) - // Seed matches add_test.go for deterministic CIDs + // +1 byte over max links threshold triggers DAG rebalancing + fileSize := fileOverMaxLinksBytes(exp) seed := seedForProfile(exp) - cidStr := node.IPFSAddDeterministic(fileSize, seed) + cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 2, len(root.Links), "expected 2 links after DAG rebalancing") - // Verify hash function on root verifyHashFunction(t, node, cidStr, exp.HashFunc) - // Verify deterministic CID if expected if exp.FileOverMaxLinksCID != "" { require.Equal(t, exp.FileOverMaxLinksCID, cidStr, "expected deterministic CID for rebalanced file") } @@ -259,7 +399,13 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri } }) - t.Run("dir-basic", func(t *testing.T) { + // Test: directory at HAMTThreshold (basic flat dir) + // A directory exactly at HAMTThreshold stays as a basic (flat) UnixFS directory. + // Threshold uses > comparison (not >=), so size == threshold stays basic. + // Size estimation method depends on profile: + // - v0-2015 "links": size = sum(nameLen + cidLen) + // - v1-2025 "block": size = serialized protobuf block size + t.Run("directory at HAMTThreshold (basic flat dir)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() @@ -270,8 +416,7 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create basic (flat) directory exactly at threshold. - // With > comparison, directory at exact threshold stays basic. + // Create basic (flat) directory exactly at threshold basicLastNameLen := exp.DirBasicLastNameLen if basicLastNameLen == 0 { basicLastNameLen = exp.DirBasicNameLen @@ -285,7 +430,7 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Verify it's a basic directory by checking UnixFS type + // Verify UnixFS type is TDirectory (1), not THAMTShard (5) fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") @@ -295,18 +440,15 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri require.Equal(t, exp.DirBasicFiles, len(root.Links), "expected basic directory with %d links", exp.DirBasicFiles) - // Verify hash function verifyHashFunction(t, node, cidStr, exp.HashFunc) // Verify size is exactly at threshold if exp.HAMTSizeEstimation == "block" { - // Block estimation: verify actual serialized block size blockSize := getBlockSize(t, node, cidStr) require.Equal(t, exp.HAMTThreshold, blockSize, "expected basic directory block size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, blockSize) } if exp.HAMTSizeEstimation == "links" { - // Links estimation: verify sum of (name_len + cid_len) for all links linksSize := 0 for _, link := range root.Links { linksSize += len(link.Name) + cidLen @@ -315,7 +457,6 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri "expected basic directory links size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, linksSize) } - // Verify deterministic CID if exp.DirBasicCID != "" { require.Equal(t, exp.DirBasicCID, cidStr, "expected deterministic CID for basic directory") } @@ -327,7 +468,11 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri } }) - t.Run("dir-hamt", func(t *testing.T) { + // Test: directory 1 byte over HAMTThreshold (HAMT sharded) + // A directory 1 byte over HAMTThreshold is converted to a HAMT sharded structure. + // HAMT distributes entries across buckets using consistent hashing. + // Root has at most HAMTFanout links (256), with entries distributed across buckets. + t.Run("directory 1 byte over HAMTThreshold (HAMT sharded)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() @@ -338,8 +483,7 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) - // Create HAMT (sharded) directory exactly +1 byte over threshold. - // With > comparison, directory over threshold becomes HAMT. + // Create HAMT (sharded) directory exactly +1 byte over threshold lastNameLen := exp.DirHAMTLastNameLen if lastNameLen == 0 { lastNameLen = exp.DirHAMTNameLen @@ -353,7 +497,7 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() - // Verify it's a HAMT directory by checking UnixFS type + // Verify UnixFS type is THAMTShard (5), not TDirectory (1) fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when over threshold") @@ -364,10 +508,8 @@ func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir stri require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, "expected HAMT directory root to have <= %d links", exp.HAMTFanout) - // Verify hash function verifyHashFunction(t, node, cidStr, exp.HashFunc) - // Verify deterministic CID if exp.DirHAMTCID != "" { require.Equal(t, exp.DirHAMTCID, cidStr, "expected deterministic CID for HAMT directory") } @@ -434,51 +576,17 @@ func getBlockSize(t *testing.T, node *harness.Node, cidStr string) int { return stat.Size } -// fileAtMaxLinksSize returns the file size that produces exactly FileMaxLinks chunks. -func fileAtMaxLinksSize(exp cidProfileExpectations) string { - switch exp.ChunkSize { - case "1MiB": - return strings.Replace(exp.ChunkSize, "1MiB", "", 1) + - string(rune('0'+exp.FileMaxLinks/1000)) + - string(rune('0'+(exp.FileMaxLinks%1000)/100)) + - string(rune('0'+(exp.FileMaxLinks%100)/10)) + - string(rune('0'+exp.FileMaxLinks%10)) + "MiB" - case "256KiB": - // 174 * 256 KiB = 44544 KiB - totalKiB := exp.FileMaxLinks * 256 - return intToStr(totalKiB) + "KiB" - default: - panic("unknown chunk size: " + exp.ChunkSize) - } +// fileAtMaxLinksBytes returns the file size in bytes that produces exactly FileMaxLinks chunks. +func fileAtMaxLinksBytes(exp cidProfileExpectations) int64 { + return int64(exp.FileMaxLinks) * int64(exp.ChunkSize) } -// fileOverMaxLinksSize returns the file size that triggers DAG rebalancing. -func fileOverMaxLinksSize(exp cidProfileExpectations) string { - switch exp.ChunkSize { - case "1MiB": - return intToStr(exp.FileMaxLinks+1) + "MiB" - case "256KiB": - // (174 + 1) * 256 KiB = 44800 KiB - totalKiB := (exp.FileMaxLinks + 1) * 256 - return intToStr(totalKiB) + "KiB" - default: - panic("unknown chunk size: " + exp.ChunkSize) - } +// fileOverMaxLinksBytes returns the file size in bytes that triggers DAG rebalancing (+1 byte over max links threshold). +func fileOverMaxLinksBytes(exp cidProfileExpectations) int64 { + return int64(exp.FileMaxLinks)*int64(exp.ChunkSize) + 1 } -func intToStr(n int) string { - if n == 0 { - return "0" - } - var digits []byte - for n > 0 { - digits = append([]byte{byte('0' + n%10)}, digits...) - n /= 10 - } - return string(digits) -} - -// seedForProfile returns the deterministic seed used in add_test.go for file tests. +// seedForProfile returns the deterministic seed used in add_test.go for file max links tests. func seedForProfile(exp cidProfileExpectations) string { switch exp.Name { case "unixfs-v0-2015", "default": @@ -490,6 +598,18 @@ func seedForProfile(exp cidProfileExpectations) string { } } +// chunkSeedForProfile returns the deterministic seed for chunk threshold tests. +func chunkSeedForProfile(exp cidProfileExpectations) string { + switch exp.Name { + case "unixfs-v0-2015", "default": + return "chunk-v0-seed" + case "unixfs-v1-2025": + return "chunk-v1-seed" + default: + return "chunk-" + exp.Name + "-seed" + } +} + // hamtSeedForProfile returns the deterministic seed for HAMT directory tests. // Uses the same seed for both under/at threshold tests to ensure consistency. func hamtSeedForProfile(exp cidProfileExpectations) string { diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index 1e08f8ed9..1ff17481a 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -77,7 +77,8 @@ func (n *Node) IPFSAddStr(content string, args ...string) string { return n.IPFSAdd(strings.NewReader(content), args...) } -// IPFSAddDeterministic produces a CID of a file of a certain size, filled with deterministically generated bytes based on some seed. +// IPFSAddDeterministic produces a CID of a file of a certain size, filled with deterministically generated bytes based on some seed. +// Size is specified as a humanize string (e.g., "256KiB", "1MiB"). // This ensures deterministic CID on the other end, that can be used in tests. func (n *Node) IPFSAddDeterministic(size string, seed string, args ...string) string { log.Debugf("node %d adding %s of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args) @@ -88,6 +89,17 @@ func (n *Node) IPFSAddDeterministic(size string, seed string, args ...string) st return n.IPFSAdd(reader, args...) } +// IPFSAddDeterministicBytes produces a CID of a file of exactly `size` bytes, filled with deterministically generated bytes based on some seed. +// Use this when exact byte precision is needed (e.g., threshold tests at T and T+1 bytes). +func (n *Node) IPFSAddDeterministicBytes(size int64, seed string, args ...string) string { + log.Debugf("node %d adding %d bytes of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args) + reader, err := DeterministicRandomReaderBytes(size, seed) + if err != nil { + panic(err) + } + return n.IPFSAdd(reader, args...) +} + func (n *Node) IPFSAdd(content io.Reader, args ...string) string { log.Debugf("node %d adding with args: %v", n.ID, args) fullArgs := []string{"add", "-q"} diff --git a/test/cli/testutils/random_deterministic.go b/test/cli/testutils/random_deterministic.go index e55404168..a4f176d24 100644 --- a/test/cli/testutils/random_deterministic.go +++ b/test/cli/testutils/random_deterministic.go @@ -27,13 +27,19 @@ func (r *randomReader) Read(p []byte) (int, error) { return int(n), nil } -// createRandomReader produces specified number of pseudo-random bytes -// from a seed. +// DeterministicRandomReader produces specified number of pseudo-random bytes +// from a seed. Size can be specified as a humanize string (e.g., "256KiB", "1MiB"). func DeterministicRandomReader(sizeStr string, seed string) (io.Reader, error) { size, err := humanize.ParseBytes(sizeStr) if err != nil { return nil, err } + return DeterministicRandomReaderBytes(int64(size), seed) +} + +// DeterministicRandomReaderBytes produces exactly `size` pseudo-random bytes +// from a seed. Use this when exact byte precision is needed. +func DeterministicRandomReaderBytes(size int64, seed string) (io.Reader, error) { // Hash the seed string to a 32-byte key for ChaCha20 key := sha256.Sum256([]byte(seed)) // Use ChaCha20 for deterministic random bytes @@ -42,5 +48,5 @@ func DeterministicRandomReader(sizeStr string, seed string) (io.Reader, error) { if err != nil { return nil, err } - return &randomReader{cipher: cipher, remaining: int64(size)}, nil + return &randomReader{cipher: cipher, remaining: size}, nil } From 800cba9cd227547fa3527119dab6f02366f2be92 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 Jan 2026 01:09:25 +0100 Subject: [PATCH 12/22] chore: update to rebased boxo and go-ipfs-cmds PRs --- docs/examples/kubo-as-a-library/go.mod | 4 ++-- docs/examples/kubo-as-a-library/go.sum | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- test/dependencies/go.mod | 4 ++-- test/dependencies/go.sum | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index c109c37f6..af5547914 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.0 + github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 @@ -85,7 +85,7 @@ require ( github.com/ipfs/go-ds-pebble v0.5.9 // indirect github.com/ipfs/go-dsqueue v0.1.2 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 1a50e42b4..e12ceda2f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.0 h1:DarrMBM46xCs6GU6Vz+AL8VUyXykqHAqZYx8mR0Oics= -github.com/ipfs/boxo v0.36.0/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -303,8 +303,8 @@ github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 h1:CvkO8fCfy6UEZDrJ021mAU2dUj3ghpOCXTVzCFqxFM8= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/go.mod b/go.mod index 7c4be05fa..6c4881309 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.0 + github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.9 github.com/ipfs/go-fs-lock v0.1.1 - github.com/ipfs/go-ipfs-cmds v0.15.0 + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 1e4be0486..42715b995 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.0 h1:DarrMBM46xCs6GU6Vz+AL8VUyXykqHAqZYx8mR0Oics= -github.com/ipfs/boxo v0.36.0/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= @@ -374,8 +374,8 @@ github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 h1:CvkO8fCfy6UEZDrJ021mAU2dUj3ghpOCXTVzCFqxFM8= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 1326c72b4..ef4b58dad 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,13 +135,13 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.0 // indirect + github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-dsqueue v0.1.2 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 8351caa9d..e2771af5b 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.0 h1:DarrMBM46xCs6GU6Vz+AL8VUyXykqHAqZYx8mR0Oics= -github.com/ipfs/boxo v0.36.0/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= +github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= @@ -314,8 +314,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs= github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402 h1:CvkO8fCfy6UEZDrJ021mAU2dUj3ghpOCXTVzCFqxFM8= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260128001524-a8594bbab402/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= From 360df63478073f36cbd2fa5850c24b95285cd836 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 28 Jan 2026 01:35:35 +0100 Subject: [PATCH 13/22] docs: add HAMT threshold fix details to changelog --- docs/changelogs/v0.40.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 3c8950556..0e691d53b 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -58,7 +58,7 @@ The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs **HAMT Threshold Fix** -HAMT directory sharding threshold changed from `>=` to `>` to match the JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. +HAMT directory sharding threshold changed from `>=` to `>` to match the GO docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. #### ๐Ÿงน Automatic cleanup of interrupted imports From ff35575e313fcb6e8065ba06ed0a2b942f38a1a1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 1 Feb 2026 20:59:58 +0100 Subject: [PATCH 14/22] feat(mfs): use Import config for CID version and hash function make MFS commands (files cp, files write, files mkdir, files chcid) respect Import.CidVersion and Import.HashFunction config settings when CLI options are not explicitly provided. also add tests for: - files write respects Import.UnixFSRawLeaves=true - single-block file: files write produces same CID as ipfs add - updated comments clarifying CID parity with ipfs add --- core/commands/files.go | 44 ++++- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/cli/files_test.go | 230 +++++++++++++++++++++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 8 files changed, 277 insertions(+), 15 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index 6d0d708fc..4076d0e3e 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -499,7 +499,12 @@ being GC'ed. return err } - prefix, err := getPrefixNew(req) + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + + prefix, err := getPrefixNew(req, &cfg.Import) if err != nil { return err } @@ -1048,7 +1053,7 @@ See '--to-files' in 'ipfs add --help' for more information. rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) } - prefix, err := getPrefixNew(req) + prefix, err := getPrefixNew(req, &cfg.Import) if err != nil { return err } @@ -1163,6 +1168,11 @@ Examples: return err } + cfg, err := n.Repo.Config() + if err != nil { + return err + } + dashp, _ := req.Options[filesParentsOptionName].(bool) dirtomake, err := checkPath(req.Arguments[0]) if err != nil { @@ -1175,7 +1185,7 @@ Examples: return err } - prefix, err := getPrefix(req) + prefix, err := getPrefix(req, &cfg.Import) if err != nil { return err } @@ -1262,7 +1272,9 @@ Change the CID version or hash function of the root node of a given path. flush, _ := req.Options[filesFlushOptionName].(bool) - prefix, err := getPrefix(req) + // Note: files chcid is for explicitly changing CID format, so we don't + // fall back to Import config here. If no options are provided, it does nothing. + prefix, err := getPrefix(req, nil) if err != nil { return err } @@ -1420,10 +1432,20 @@ func removePath(filesRoot *mfs.Root, path string, force bool, dashr bool) error return pdir.Flush() } -func getPrefixNew(req *cmds.Request) (cid.Builder, error) { +func getPrefixNew(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) { cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int) hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string) + // Fall back to Import config if CLI options not set + if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() { + cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion)) + cidVerSet = true + } + if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() { + hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction) + hashFunSet = true + } + if !cidVerSet && !hashFunSet { return nil, nil } @@ -1449,10 +1471,20 @@ func getPrefixNew(req *cmds.Request) (cid.Builder, error) { return &prefix, nil } -func getPrefix(req *cmds.Request) (cid.Builder, error) { +func getPrefix(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) { cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int) hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string) + // Fall back to Import config if CLI options not set + if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() { + cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion)) + cidVerSet = true + } + if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() { + hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction) + hashFunSet = true + } + if !cidVerSet && !hashFunSet { return nil, nil } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 98560a4ab..7e33b1102 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a + github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index a64b0d35c..fdd5ab799 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index f8602d8c4..7bb8b8aa3 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a + github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 1d784d776..01c43962a 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/cli/files_test.go b/test/cli/files_test.go index c2d226f90..bb817f2c8 100644 --- a/test/cli/files_test.go +++ b/test/cli/files_test.go @@ -4,8 +4,10 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" @@ -459,3 +461,231 @@ func TestFilesChroot(t *testing.T) { assert.Contains(t, res.Stderr.String(), "opening repo") }) } + +// TestFilesMFSImportConfig tests that MFS operations respect Import.* configuration settings. +// These tests verify that `ipfs files` commands use the same import settings as `ipfs add`. +func TestFilesMFSImportConfig(t *testing.T) { + t.Parallel() + + t.Run("files write respects Import.CidVersion=1", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + // Write file via MFS + tempFile := filepath.Join(node.Dir, "test.txt") + require.NoError(t, os.WriteFile(tempFile, []byte("hello"), 0644)) + node.IPFS("files", "write", "--create", "/test.txt", tempFile) + + // Get CID of written file + cidStr := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() + + // Verify CIDv1 format (base32, starts with "b") + require.True(t, strings.HasPrefix(cidStr, "b"), "expected CIDv1 (starts with b), got: %s", cidStr) + }) + + t.Run("files write respects Import.UnixFSRawLeaves=true", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + }) + node.StartDaemon() + defer node.StopDaemon() + + tempFile := filepath.Join(node.Dir, "test.txt") + require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) + node.IPFS("files", "write", "--create", "/test.txt", tempFile) + + cidStr := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() + codec := node.IPFS("cid", "format", "-f", "%c", cidStr).Stdout.Trimmed() + require.Equal(t, "raw", codec, "expected raw codec for small file with raw leaves") + }) + + // This test verifies CID parity for single-block files only. + // Multi-block files will have different CIDs because MFS uses trickle DAG layout + // while 'ipfs add' uses balanced DAG layout. See "files write vs add for multi-block" test. + t.Run("single-block file: files write produces same CID as ipfs add", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + }) + node.StartDaemon() + defer node.StopDaemon() + + tempFile := filepath.Join(node.Dir, "test.txt") + require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) + node.IPFS("files", "write", "--create", "/test.txt", tempFile) + + mfsCid := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() + addCid := node.IPFSAddStr("hello world") + require.Equal(t, addCid, mfsCid, "MFS write should produce same CID as ipfs add for single-block files") + }) + + t.Run("files mkdir respects Import.CidVersion=1", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + node.IPFS("files", "mkdir", "/testdir") + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + + // Verify CIDv1 format + require.True(t, strings.HasPrefix(cidStr, "b"), "expected CIDv1 (starts with b), got: %s", cidStr) + }) + + t.Run("MFS subdirectory becomes HAMT when exceeding threshold", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + // Use small threshold for faster testing + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + defer node.StopDaemon() + + node.IPFS("files", "mkdir", "/bigdir") + + content := "x" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + // Add enough files to exceed 1KiB threshold + for i := 0; i < 25; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/bigdir/file%02d", i), tempFile) + } + + cidStr := node.IPFS("files", "stat", "--hash", "/bigdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory") + }) + + t.Run("MFS root directory becomes HAMT when exceeding threshold", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + defer node.StopDaemon() + + content := "x" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + // Add files directly to root / + for i := 0; i < 25; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/file%02d", i), tempFile) + } + + cidStr := node.IPFS("files", "stat", "--hash", "/").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected MFS root to become HAMT") + }) + + t.Run("MFS directory reverts from HAMT to basic when items removed", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + defer node.StopDaemon() + + node.IPFS("files", "mkdir", "/testdir") + + content := "x" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + // Add files to exceed threshold + for i := 0; i < 25; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%02d", i), tempFile) + } + + // Verify it became HAMT + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "should be HAMT after adding many files") + + // Remove files to get back below threshold + for i := 0; i < 20; i++ { + node.IPFS("files", "rm", fmt.Sprintf("/testdir/file%02d", i)) + } + + // Verify it reverted to basic directory + cidStr = node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err = node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TDirectory, fsType, "should revert to basic directory after removing files") + }) + + // Note: 'files write' produces DIFFERENT CIDs than 'ipfs add' for multi-block files because + // MFS uses trickle DAG layout while 'ipfs add' uses balanced DAG layout. + // Single-block files produce the same CID (tested above in "single-block file: files write..."). + // For multi-block CID compatibility with 'ipfs add', use 'ipfs add --to-files' instead. + + t.Run("files cp preserves original CID", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + }) + node.StartDaemon() + defer node.StopDaemon() + + // Add file via ipfs add + originalCid := node.IPFSAddStr("hello world") + + // Copy to MFS + node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", originalCid), "/copied.txt") + + // Verify CID is preserved + mfsCid := node.IPFS("files", "stat", "--hash", "/copied.txt").Stdout.Trimmed() + require.Equal(t, originalCid, mfsCid, "files cp should preserve original CID") + }) + + t.Run("add --to-files respects Import config", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + }) + node.StartDaemon() + defer node.StopDaemon() + + // Create temp file + tempFile := filepath.Join(node.Dir, "test.txt") + require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) + + // Add with --to-files + addCid := node.IPFS("add", "-Q", "--to-files=/added.txt", tempFile).Stdout.Trimmed() + + // Verify MFS file has same CID + mfsCid := node.IPFS("files", "stat", "--hash", "/added.txt").Stdout.Trimmed() + require.Equal(t, addCid, mfsCid) + + // Should be CIDv1 raw leaf + codec := node.IPFS("cid", "format", "-f", "%c", mfsCid).Stdout.Trimmed() + require.Equal(t, "raw", codec) + }) +} diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index dfe079150..7c0d5d79b 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a // indirect + github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 35f9e6618..e75a95622 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a h1:/NV/SudfOSqEbJylxiKyh4vm2yQZHPfnOGlucDURIz0= -github.com/ipfs/boxo v0.36.1-0.20260128000516-56cf0aecdc1a/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= +github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 506cc6e70fb3bf094c967787ac0cd187c2da3adb Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 2 Feb 2026 02:00:02 +0100 Subject: [PATCH 15/22] feat(files): wire Import.UnixFSChunker and UnixFSDirectoryMaxLinks to MFS `ipfs files` commands now respect these Import.* config options: - UnixFSChunker: configures chunk size for `files write` - UnixFSDirectoryMaxLinks: triggers HAMT sharding in `files mkdir` - UnixFSHAMTDirectorySizeEstimation: controls size estimation mode previously, MFS used hardcoded defaults ignoring user config. changes: - config/import.go: add UnixFSSplitterFunc() returning chunk.SplitterGen - core/node/core.go: pass chunker, maxLinks, sizeEstimationMode to mfs.NewRoot() via new boxo RootOption API - core/commands/files.go: pass maxLinks and sizeEstimationMode to mfs.Mkdir() and ensureContainingDirectoryExists(); document that UnixFSFileMaxLinks doesn't apply to files write (trickle DAG limitation) - test/cli/files_test.go: add tests for UnixFSDirectoryMaxLinks and UnixFSChunker, including CID parity test with `ipfs add --trickle` related: boxo@54e044f1b265 --- config/import.go | 40 +++++++++-- core/commands/files.go | 34 +++++++--- core/node/core.go | 15 +++- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 4 +- go.sum | 2 - test/cli/files_test.go | 94 ++++++++++++++++++++++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 10 files changed, 175 insertions(+), 26 deletions(-) diff --git a/config/import.go b/config/import.go index 2c88a12f3..8c40d7d1e 100644 --- a/config/import.go +++ b/config/import.go @@ -2,11 +2,13 @@ package config import ( "fmt" + "io" "strconv" "strings" + chunk "github.com/ipfs/boxo/chunker" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" - "github.com/ipfs/boxo/ipld/unixfs/io" + uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/verifcid" mh "github.com/multiformats/go-multihash" ) @@ -47,7 +49,7 @@ const ( var ( DefaultUnixFSFileMaxLinks = int64(helpers.DefaultLinksPerBlock) DefaultUnixFSDirectoryMaxLinks = int64(0) - DefaultUnixFSHAMTDirectoryMaxFanout = int64(io.DefaultShardWidth) + DefaultUnixFSHAMTDirectoryMaxFanout = int64(uio.DefaultShardWidth) ) // Import configures the default options for ingesting data. This affects commands @@ -222,15 +224,39 @@ func isValidChunker(chunker string) bool { } // HAMTSizeEstimationMode returns the boxo SizeEstimationMode based on the config value. -func (i *Import) HAMTSizeEstimationMode() io.SizeEstimationMode { +func (i *Import) HAMTSizeEstimationMode() uio.SizeEstimationMode { switch i.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) { case HAMTSizeEstimationLinks: - return io.SizeEstimationLinks + return uio.SizeEstimationLinks case HAMTSizeEstimationBlock: - return io.SizeEstimationBlock + return uio.SizeEstimationBlock case HAMTSizeEstimationDisabled: - return io.SizeEstimationDisabled + return uio.SizeEstimationDisabled default: - return io.SizeEstimationLinks + return uio.SizeEstimationLinks + } +} + +// UnixFSSplitterFunc returns a SplitterGen function based on Import.UnixFSChunker. +// The returned function creates a Splitter for the configured chunking strategy. +// The chunker string is parsed once when this method is called, not on each use. +func (i *Import) UnixFSSplitterFunc() chunk.SplitterGen { + chunkerStr := i.UnixFSChunker.WithDefault(DefaultUnixFSChunker) + + // Parse size-based chunker (most common case) and return optimized generator + if sizeStr, ok := strings.CutPrefix(chunkerStr, "size-"); ok { + if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 { + return chunk.SizeSplitterGen(size) + } + } + + // For other chunker types (rabin, buzhash) or invalid config, + // fall back to parsing per-use (these are rare cases) + return func(r io.Reader) chunk.Splitter { + s, err := chunk.FromString(r, chunkerStr) + if err != nil { + return chunk.DefaultSplitter(r) + } + return s } } diff --git a/core/commands/files.go b/core/commands/files.go index 4076d0e3e..252ba467a 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -28,6 +28,7 @@ import ( offline "github.com/ipfs/boxo/exchange/offline" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" + uio "github.com/ipfs/boxo/ipld/unixfs/io" mfs "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" @@ -555,7 +556,9 @@ being GC'ed. mkParents, _ := req.Options[filesParentsOptionName].(bool) if mkParents { - err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix) + maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) + sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() + err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix, maxDirLinks, &sizeEstimationMode) if err != nil { return err } @@ -994,9 +997,13 @@ stat' on the file or any of its ancestors. WARNING: The CID produced by 'files write' will be different from 'ipfs add' because -'ipfs file write' creates a trickle-dag optimized for append-only operations +'ipfs file write' creates a trickle-dag optimized for append-only operations. See '--trickle' in 'ipfs add --help' for more information. +NOTE: The 'Import.UnixFSFileMaxLinks' config option does not apply to this command. +Trickle DAG has a fixed internal structure optimized for append operations. +To use configurable max-links, use 'ipfs add' with balanced DAG layout. + If you want to add a file without modifying an existing one, use 'ipfs add' with '--to-files': @@ -1064,7 +1071,9 @@ See '--to-files' in 'ipfs add --help' for more information. } if mkParents { - err := ensureContainingDirectoryExists(nd.FilesRoot, path, prefix) + maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) + sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() + err := ensureContainingDirectoryExists(nd.FilesRoot, path, prefix, maxDirLinks, &sizeEstimationMode) if err != nil { return err } @@ -1191,10 +1200,15 @@ Examples: } root := n.FilesRoot + maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) + sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() + err = mfs.Mkdir(root, dirtomake, mfs.MkdirOpts{ - Mkparents: dashp, - Flush: flush, - CidBuilder: prefix, + Mkparents: dashp, + Flush: flush, + CidBuilder: prefix, + MaxLinks: maxDirLinks, + SizeEstimationMode: &sizeEstimationMode, }) return err @@ -1510,7 +1524,7 @@ func getPrefix(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) return &prefix, nil } -func ensureContainingDirectoryExists(r *mfs.Root, path string, builder cid.Builder) error { +func ensureContainingDirectoryExists(r *mfs.Root, path string, builder cid.Builder, maxLinks int, sizeEstimationMode *uio.SizeEstimationMode) error { dirtomake := gopath.Dir(path) if dirtomake == "/" { @@ -1518,8 +1532,10 @@ func ensureContainingDirectoryExists(r *mfs.Root, path string, builder cid.Build } return mfs.Mkdir(r, dirtomake, mfs.MkdirOpts{ - Mkparents: true, - CidBuilder: builder, + Mkparents: true, + CidBuilder: builder, + MaxLinks: maxLinks, + SizeEstimationMode: sizeEstimationMode, }) } diff --git a/core/node/core.go b/core/node/core.go index 06e786f1f..fd23cc16b 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -243,7 +243,20 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo prov = nil } - root, err := mfs.NewRoot(ctx, dag, nd, pf, prov) + // Get configured settings from Import config + cfg, err := repo.Config() + if err != nil { + return nil, fmt.Errorf("failed to get config: %w", err) + } + chunkerGen := cfg.Import.UnixFSSplitterFunc() + maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) + sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() + + root, err := mfs.NewRoot(ctx, dag, nd, pf, prov, + mfs.WithChunker(chunkerGen), + mfs.WithMaxLinks(maxDirLinks), + mfs.WithSizeEstimationMode(sizeEstimationMode), + ) if err != nil { return nil, fmt.Errorf("failed to initialize MFS root from %s stored at %s: %w. "+ "If corrupted, use 'ipfs files chroot' to reset (see --help)", nd.Cid(), FilesRootDatastoreKey, err) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 7e33b1102..ba63471a4 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 + github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index fdd5ab799..241f89289 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 h1:zqISQlY0hN/IQsNB5adpPSpuqcgRwQnboxv6ArxXt5k= +github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 7bb8b8aa3..f7c91f275 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 + github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 @@ -279,3 +279,5 @@ exclude ( github.com/ipfs/go-ipfs-cmds v2.0.1+incompatible github.com/libp2p/go-libp2p v6.0.23+incompatible ) + +replace github.com/ipfs/boxo => ../boxo diff --git a/go.sum b/go.sum index 01c43962a..0c6a3b906 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,6 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/cli/files_test.go b/test/cli/files_test.go index bb817f2c8..6af1bc63d 100644 --- a/test/cli/files_test.go +++ b/test/cli/files_test.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -688,4 +689,97 @@ func TestFilesMFSImportConfig(t *testing.T) { codec := node.IPFS("cid", "format", "-f", "%c", mfsCid).Stdout.Trimmed() require.Equal(t, "raw", codec) }) + + t.Run("files mkdir respects Import.UnixFSDirectoryMaxLinks", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + // Set low link threshold to trigger HAMT sharding at 5 links + cfg.Import.UnixFSDirectoryMaxLinks = *config.NewOptionalInteger(5) + // Also need size estimation enabled for switching to work + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + defer node.StopDaemon() + + // Create directory with 6 files (exceeds max 5 links) + node.IPFS("files", "mkdir", "/testdir") + + content := "x" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + for i := 0; i < 6; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) + } + + // Verify directory became HAMT sharded + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory after exceeding UnixFSDirectoryMaxLinks") + }) + + t.Run("files write respects Import.UnixFSChunker", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + cfg.Import.UnixFSChunker = *config.NewOptionalString("size-1024") // 1KB chunks + }) + node.StartDaemon() + defer node.StopDaemon() + + // Create file larger than chunk size (3KB) + data := make([]byte, 3*1024) + for i := range data { + data[i] = byte(i % 256) + } + tempFile := filepath.Join(node.Dir, "large.bin") + require.NoError(t, os.WriteFile(tempFile, data, 0644)) + + node.IPFS("files", "write", "--create", "/large.bin", tempFile) + + // Verify chunking: 3KB file with 1KB chunks should have multiple child blocks + cidStr := node.IPFS("files", "stat", "--hash", "/large.bin").Stdout.Trimmed() + dagStatJSON := node.IPFS("dag", "stat", "--enc=json", cidStr).Stdout.Trimmed() + var dagStat struct { + UniqueBlocks int `json:"UniqueBlocks"` + } + require.NoError(t, json.Unmarshal([]byte(dagStatJSON), &dagStat)) + // With 1KB chunks on a 3KB file, we expect 4 blocks (3 leaf + 1 root) + assert.Greater(t, dagStat.UniqueBlocks, 1, "expected more than 1 block with 1KB chunker on 3KB file") + }) + + t.Run("files write with custom chunker produces same CID as ipfs add --trickle", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + cfg.Import.UnixFSRawLeaves = config.True + cfg.Import.UnixFSChunker = *config.NewOptionalString("size-512") + }) + node.StartDaemon() + defer node.StopDaemon() + + // Create test data (2KB to get multiple chunks) + data := make([]byte, 2048) + for i := range data { + data[i] = byte(i % 256) + } + tempFile := filepath.Join(node.Dir, "test.bin") + require.NoError(t, os.WriteFile(tempFile, data, 0644)) + + // Add via MFS + node.IPFS("files", "write", "--create", "/test.bin", tempFile) + mfsCid := node.IPFS("files", "stat", "--hash", "/test.bin").Stdout.Trimmed() + + // Add via ipfs add with same chunker and trickle (MFS always uses trickle) + addCid := node.IPFS("add", "-Q", "--chunker=size-512", "--trickle", tempFile).Stdout.Trimmed() + + // CIDs should match when using same chunker + trickle layout + require.Equal(t, addCid, mfsCid, "MFS and add --trickle should produce same CID with matching chunker") + }) } diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 7c0d5d79b..50d81099b 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 // indirect + github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index e75a95622..67b43843a 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779 h1:hWjjMiiu6aEDXqJmoF3opHrtT3EXwivLE2N0f87yDDU= -github.com/ipfs/boxo v0.36.1-0.20260201194832-c910c48ea779/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 h1:zqISQlY0hN/IQsNB5adpPSpuqcgRwQnboxv6ArxXt5k= +github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 9bb4a4e22c1a7f4323fd981196986420ceb745d5 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 2 Feb 2026 04:08:03 +0100 Subject: [PATCH 16/22] feat(files): wire Import.UnixFSHAMTDirectoryMaxFanout and UnixFSHAMTDirectorySizeThreshold wire remaining HAMT config options to MFS root: - Import.UnixFSHAMTDirectoryMaxFanout via mfs.WithMaxHAMTFanout - Import.UnixFSHAMTDirectorySizeThreshold via mfs.WithHAMTShardingSize add CLI tests: - files mkdir respects Import.UnixFSHAMTDirectoryMaxFanout - files mkdir respects Import.UnixFSHAMTDirectorySizeThreshold - config change takes effect after daemon restart add UnixFSHAMTFanout() helper to test harness update boxo to ac97424d99ab90e097fc7c36f285988b596b6f05 --- core/node/core.go | 4 + docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 4 +- go.sum | 2 + test/cli/files_test.go | 113 +++++++++++++++++++++++++ test/cli/harness/pbinspect.go | 30 +++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 9 files changed, 156 insertions(+), 9 deletions(-) diff --git a/core/node/core.go b/core/node/core.go index fd23cc16b..0b2af81c3 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -250,11 +250,15 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo } chunkerGen := cfg.Import.UnixFSSplitterFunc() maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) + maxHAMTFanout := int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)) + hamtShardingSize := int(cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold)) sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() root, err := mfs.NewRoot(ctx, dag, nd, pf, prov, mfs.WithChunker(chunkerGen), mfs.WithMaxLinks(maxDirLinks), + mfs.WithMaxHAMTFanout(maxHAMTFanout), + mfs.WithHAMTShardingSize(hamtShardingSize), mfs.WithSizeEstimationMode(sizeEstimationMode), ) if err != nil { diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index ba63471a4..596f7cd83 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 + github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 241f89289..0601423b8 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 h1:zqISQlY0hN/IQsNB5adpPSpuqcgRwQnboxv6ArxXt5k= -github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index f7c91f275..dcc21f033 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 + github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 @@ -279,5 +279,3 @@ exclude ( github.com/ipfs/go-ipfs-cmds v2.0.1+incompatible github.com/libp2p/go-libp2p v6.0.23+incompatible ) - -replace github.com/ipfs/boxo => ../boxo diff --git a/go.sum b/go.sum index 0c6a3b906..6745ce6dc 100644 --- a/go.sum +++ b/go.sum @@ -338,6 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/cli/files_test.go b/test/cli/files_test.go index 6af1bc63d..5187c8313 100644 --- a/test/cli/files_test.go +++ b/test/cli/files_test.go @@ -782,4 +782,117 @@ func TestFilesMFSImportConfig(t *testing.T) { // CIDs should match when using same chunker + trickle layout require.Equal(t, addCid, mfsCid, "MFS and add --trickle should produce same CID with matching chunker") }) + + t.Run("files mkdir respects Import.UnixFSHAMTDirectoryMaxFanout", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + // Use non-default fanout of 64 (default is 256) + cfg.Import.UnixFSHAMTDirectoryMaxFanout = *config.NewOptionalInteger(64) + // Set low link threshold to trigger HAMT at 5 links + cfg.Import.UnixFSDirectoryMaxLinks = *config.NewOptionalInteger(5) + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("disabled") + }) + node.StartDaemon() + defer node.StopDaemon() + + node.IPFS("files", "mkdir", "/testdir") + + content := "x" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + // Add 6 files (exceeds MaxLinks=5) to trigger HAMT + for i := 0; i < 6; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) + } + + // Verify directory became HAMT + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory") + + // Verify the HAMT uses the custom fanout (64) by inspecting the UnixFS Data field. + fanout, err := node.UnixFSHAMTFanout(cidStr) + require.NoError(t, err) + require.Equal(t, uint64(64), fanout, "expected HAMT fanout 64") + }) + + t.Run("files mkdir respects Import.UnixFSHAMTDirectorySizeThreshold", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + // Use very small threshold (100 bytes) to trigger HAMT quickly + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("100B") + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + defer node.StopDaemon() + + node.IPFS("files", "mkdir", "/testdir") + + content := "test content" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + + // Add 3 files - each link adds ~40-50 bytes, so 3 should exceed 100B threshold + for i := 0; i < 3; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) + } + + // Verify directory became HAMT due to size threshold + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory after exceeding size threshold") + }) + + t.Run("config change takes effect after daemon restart", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + + // Start with high threshold (won't trigger HAMT) + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("256KiB") + cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") + }) + node.StartDaemon() + + // Create directory with some files + node.IPFS("files", "mkdir", "/testdir") + content := "test" + tempFile := filepath.Join(node.Dir, "content.txt") + require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) + for i := 0; i < 3; i++ { + node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) + } + + // Verify it's still a basic directory (threshold not exceeded) + cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err := node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.TDirectory, fsType, "should be basic directory with high threshold") + + // Stop daemon + node.StopDaemon() + + // Change config to use very low threshold + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("100B") + }) + + // Restart daemon + node.StartDaemon() + defer node.StopDaemon() + + // Add one more file - this should trigger HAMT conversion with new threshold + node.IPFS("files", "write", "--create", "/testdir/file3.txt", tempFile) + + // Verify it became HAMT (new threshold applied) + cidStr = node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() + fsType, err = node.UnixFSDataType(cidStr) + require.NoError(t, err) + require.Equal(t, ft.THAMTShard, fsType, "should be HAMT after daemon restart with lower threshold") + }) } diff --git a/test/cli/harness/pbinspect.go b/test/cli/harness/pbinspect.go index 6210e7bed..0ebcdd8b6 100644 --- a/test/cli/harness/pbinspect.go +++ b/test/cli/harness/pbinspect.go @@ -44,6 +44,36 @@ func (n *Node) UnixFSDataType(cid string) (pb.Data_DataType, error) { return fsNode.Type(), nil } +// UnixFSHAMTFanout returns the fanout value for a HAMT shard directory. +// This is only valid for HAMT shards (THAMTShard type). +func (n *Node) UnixFSHAMTFanout(cid string) (uint64, error) { + log.Debugf("node %d block get %s for fanout", n.ID, cid) + + var blockData bytes.Buffer + res := n.Runner.MustRun(RunRequest{ + Path: n.IPFSBin, + Args: []string{"block", "get", cid}, + CmdOpts: []CmdOpt{RunWithStdout(&blockData)}, + }) + if res.Err != nil { + return 0, res.Err + } + + // Parse dag-pb block + protoNode, err := mdag.DecodeProtobuf(blockData.Bytes()) + if err != nil { + return 0, err + } + + // Parse UnixFS data + fsNode, err := ft.FSNodeFromBytes(protoNode.Data()) + if err != nil { + return 0, err + } + + return fsNode.Fanout(), nil +} + // InspectPBNode uses dag-json output of 'ipfs dag get' to inspect // "Logical Format" of DAG-PB as defined in // https://web.archive.org/web/20250403194752/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 50d81099b..6dc40cd66 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 // indirect + github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 67b43843a..7942c95e4 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265 h1:zqISQlY0hN/IQsNB5adpPSpuqcgRwQnboxv6ArxXt5k= -github.com/ipfs/boxo v0.36.1-0.20260202005650-54e044f1b265/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= +github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From c2d414f8fa7fd5d2a39c1ba85ca71d201947576b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 00:30:19 +0100 Subject: [PATCH 17/22] fix(mfs): single-block files in CIDv1 dirs now produce raw CIDs problem: `ipfs files write` in CIDv1 directories wrapped single-block files in dag-pb even when raw-leaves was enabled, producing different CIDs than `ipfs add --raw-leaves` for the same content. fix: boxo now collapses single-block ProtoNode wrappers (with no metadata) to RawNode in DagModifier.GetNode(). files with mtime/mode stay as dag-pb since raw blocks cannot store UnixFS metadata. also fixes sparse file writes where writing past EOF would lose data because expandSparse didn't update the internal node pointer. updates boxo to v0.36.1-0.20260203003133-7884ae23aaff updates t0250-files-api.sh test hashes to match new behavior --- docs/changelogs/v0.40.md | 6 ++++- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- test/sharness/t0250-files-api.sh | 31 +++++++++++++++++--------- 8 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index a8271910e..735bf014c 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -59,9 +59,13 @@ The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs - `--hidden` / `-H` includes hidden files (default: false) - `--trickle` implicit default can be adjusted via `Import.UnixFSDAGLayout` +**`ipfs files write` fix for CIDv1 directories** + +When writing to MFS directories that use CIDv1 (via `--cid-version=1` or `ipfs files chcid`), single-block files now produce raw block CIDs (like `bafkrei...`), matching the behavior of `ipfs add --raw-leaves`. Previously, MFS would wrap single-block files in dag-pb even when raw leaves were enabled. CIDv0 directories continue to use dag-pb. + **HAMT Threshold Fix** -HAMT directory sharding threshold changed from `>=` to `>` to match the GO docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. +HAMT directory sharding threshold changed from `>=` to `>` to match the Go docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. #### ๐Ÿงน Automatic cleanup of interrupted imports diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 596f7cd83..c6c3ab14b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 0601423b8..cdb00389d 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index dcc21f033..6cc5e5c43 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 6745ce6dc..95e53c165 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 6dc40cd66..c5a2a5af5 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab // indirect + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 7942c95e4..755148e26 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index b86ee56f5..ad9ca5f81 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -786,6 +786,7 @@ tests_for_files_api() { test_expect_success "can create some files for testing ($EXTRA)" ' create_files ' + # default: CIDv0, dag-pb for all files (no raw-leaves) ROOT_HASH=QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt CATS_HASH=Qma88m8ErTGkZHbBWGqy1C7VmEmX8wwNDWNpGyCaNmEgwC FILE_HASH=QmQdQt9qooenjeaNhiKHF3hBvmNteB4MQBtgu3jxgf9c7i @@ -796,20 +797,23 @@ tests_for_files_api() { create_files --raw-leaves ' + # partial raw-leaves: initial files created with --raw-leaves, test ops without if [ "$EXTRA" = "with-daemon" ]; then ROOT_HASH=QmTpKiKcAj4sbeesN6vrs5w3QeVmd4QmGpxRL81hHut4dZ CATS_HASH=QmPhPkmtUGGi8ySPHoPu1qbfryLJKKq1GYxpgLyyCruvGe test_files_api "($EXTRA, partial raw-leaves)" fi - ROOT_HASH=QmW3dMSU6VNd1mEdpk9S3ZYRuR1YwwoXjGaZhkyK6ru9YU - CATS_HASH=QmPqWDEg7NoWRX8Y4vvYjZtmdg5umbfsTQ9zwNr12JoLmt - FILE_HASH=QmRCgHeoKxCqK2Es6M6nPUDVWz19yNQPnsXGsXeuTkSKpN - TRUNC_HASH=QmckstrVxJuecVD1FHUiURJiU9aPURZWJieeBVHJPACj8L + # raw-leaves: single-block files become RawNode (CIDv1), dirs stay CIDv0 + ROOT_HASH=QmTHzLiSouBHVTssS8xRzmfWGAvTGhPEjtPdB6pWMQdxJX + CATS_HASH=QmPJkzbCoBuL379TbHgwF1YbVHnKgiDa5bjqYhe6Lovdms + FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm + TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e test_files_api "($EXTRA, raw-leaves)" '' --raw-leaves - ROOT_HASH=QmageRWxC7wWjPv5p36NeAgBAiFdBHaNfxAehBSwzNech2 - CATS_HASH=bafybeig4cpvfu2qwwo3u4ffazhqdhyynfhnxqkzvbhrdbamauthf5mfpuq + # cidv1 for mkdir: different from raw-leaves since mkdir forces CIDv1 dirs + ROOT_HASH=QmTLdTaZNj8Mvq1cgYup59ZFJFv1KxptouFSZUZKeq7X3z + CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e if [ "$EXTRA" = "with-daemon" ]; then @@ -823,8 +827,10 @@ tests_for_files_api() { test_cmp hash_expect hash_actual ' - ROOT_HASH=bafybeifxnoetaa2jetwmxubv3gqiyaknnujwkkkhdeua63kulm63dcr5wu - test_files_api "($EXTRA, cidv1 root)" + # cidv1 root: root upgraded to CIDv1 via chcid, all new dirs/files also CIDv1 + ROOT_HASH=bafybeickjecu37qv6ue54ofk3n4rpm4g4abuofz7yc4qn4skffy263kkou + CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm + test_files_api "($EXTRA, cidv1 root)" if [ "$EXTRA" = "with-daemon" ]; then test_expect_success "can update root hash to blake2b-256" ' @@ -833,8 +839,9 @@ tests_for_files_api() { ipfs files stat --hash / > hash_actual && test_cmp hash_expect hash_actual ' - ROOT_HASH=bafykbzaceb6jv27itwfun6wsrbaxahpqthh5be2bllsjtb3qpmly3vji4mlfk - CATS_HASH=bafykbzacebhpn7rtcjjc5oa4zgzivhs7a6e2tq4uk4px42bubnmhpndhqtjig + # blake2b-256 root: using blake2b-256 hash instead of sha2-256 + ROOT_HASH=bafykbzaceaebvwrjdw5rfhqqh5miaq3g42yybnrw3kxxxx43ggyttm6xn2zek + CATS_HASH=bafykbzaceaqvpxs3dfl7su6744jgyvifbusow2tfixdy646chasdwyz2boagc FILE_HASH=bafykbzaceca45w2i3o3q3ctqsezdv5koakz7sxsw37ygqjg4w54m2bshzevxy TRUNC_HASH=bafykbzaceadeu7onzmlq7v33ytjpmo37rsqk2q6mzeqf5at55j32zxbcdbwig test_files_api "($EXTRA, blake2b-256 root)" @@ -866,10 +873,12 @@ test_expect_success "enable sharding in config" ' test_launch_ipfs_daemon_without_network +# sharding cidv0: HAMT-sharded directory with 100 files, CIDv0 SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se test_sharding "(cidv0)" -SHARD_HASH=bafybeib46tpawg2d2hhlmmn2jvgio33wqkhlehxrem7wbfvqqikure37rm +# sharding cidv1: HAMT-sharded directory with 100 files, CIDv1 +SHARD_HASH=bafybeiaulcf7c46pqg3tkud6dsvbgvlnlhjuswcwtfhxts5c2kuvmh5keu test_sharding "(cidv1 root)" "--cid-version=1" test_kill_ipfs_daemon From 32b629cc2226209477649c85e4c8fb4fa9690cc1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 04:35:20 +0100 Subject: [PATCH 18/22] chore(test): use Go 1.22+ range-over-int syntax --- test/cli/files_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cli/files_test.go b/test/cli/files_test.go index 5187c8313..ed92cdb30 100644 --- a/test/cli/files_test.go +++ b/test/cli/files_test.go @@ -564,7 +564,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add enough files to exceed 1KiB threshold - for i := 0; i < 25; i++ { + for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/bigdir/file%02d", i), tempFile) } @@ -589,7 +589,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add files directly to root / - for i := 0; i < 25; i++ { + for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/file%02d", i), tempFile) } @@ -616,7 +616,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add files to exceed threshold - for i := 0; i < 25; i++ { + for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%02d", i), tempFile) } @@ -627,7 +627,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.Equal(t, ft.THAMTShard, fsType, "should be HAMT after adding many files") // Remove files to get back below threshold - for i := 0; i < 20; i++ { + for i := range 20 { node.IPFS("files", "rm", fmt.Sprintf("/testdir/file%02d", i)) } @@ -710,7 +710,7 @@ func TestFilesMFSImportConfig(t *testing.T) { tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) - for i := 0; i < 6; i++ { + for i := range 6 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } @@ -803,7 +803,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add 6 files (exceeds MaxLinks=5) to trigger HAMT - for i := 0; i < 6; i++ { + for i := range 6 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } @@ -837,7 +837,7 @@ func TestFilesMFSImportConfig(t *testing.T) { require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add 3 files - each link adds ~40-50 bytes, so 3 should exceed 100B threshold - for i := 0; i < 3; i++ { + for i := range 3 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } @@ -864,7 +864,7 @@ func TestFilesMFSImportConfig(t *testing.T) { content := "test" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) - for i := 0; i < 3; i++ { + for i := range 3 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } From 7fdb2c24236ef478b8f9b0532fb9a3ade3476df0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 18:11:05 +0100 Subject: [PATCH 19/22] chore: update boxo to c6829fe26860 - fix typo in files write help text - update boxo with CI fixes (gofumpt, race condition in test) --- core/commands/files.go | 2 +- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index 252ba467a..d9ab9e980 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -997,7 +997,7 @@ stat' on the file or any of its ancestors. WARNING: The CID produced by 'files write' will be different from 'ipfs add' because -'ipfs file write' creates a trickle-dag optimized for append-only operations. +'ipfs files write' creates a trickle-dag optimized for append-only operations. See '--trickle' in 'ipfs add --help' for more information. NOTE: The 'Import.UnixFSFileMaxLinks' config option does not apply to this command. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index c6c3ab14b..42e4e8c87 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff + github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index cdb00389d..1e502e80c 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 6cc5e5c43..51e629478 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff + github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 95e53c165..19452e5bd 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index c5a2a5af5..a8da9545f 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff // indirect + github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 755148e26..ae8b3a03d 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= -github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= +github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From 44793f57318f5ffdc4593830d00944997e31cdbf Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 18:44:00 +0100 Subject: [PATCH 20/22] chore: update go-ipfs-cmds to 192ec9d15c1f includes binary content types fix: gzip, zip, vnd.ipld.car, vnd.ipld.raw, vnd.ipfs.ipns-record --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 79a46330b..a880716d2 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -85,7 +85,7 @@ require ( github.com/ipfs/go-ds-pebble v0.5.9 // indirect github.com/ipfs/go-dsqueue v0.1.2 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 5e0064604..446e2a9e4 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -303,8 +303,8 @@ github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 h1:FnQqL92YxPX08/dcqE4cCSqEzwVGSdj2wprWHX+cUtM= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483/go.mod h1:YmhRbpaLKg40i9Ogj2+L41tJ+8x50fF8u1FJJD/WNhc= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f h1:6bm6WhriotbESJkTWQxVuiBiXfynOTU5oq3j1bJyNo4= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/go.mod b/go.mod index 1b87706ff..70e3f313c 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.9 github.com/ipfs/go-fs-lock v0.1.1 - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 0623ea3f4..2c2640384 100644 --- a/go.sum +++ b/go.sum @@ -373,8 +373,8 @@ github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 h1:FnQqL92YxPX08/dcqE4cCSqEzwVGSdj2wprWHX+cUtM= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483/go.mod h1:YmhRbpaLKg40i9Ogj2+L41tJ+8x50fF8u1FJJD/WNhc= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f h1:6bm6WhriotbESJkTWQxVuiBiXfynOTU5oq3j1bJyNo4= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 15173dff0..53a6ea263 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -141,7 +141,7 @@ require ( github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-dsqueue v0.1.2 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 57b7790a6..0b0cfab7a 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -314,8 +314,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.1.2 h1:jBMsgvT9Pj9l3cqI0m5jYpW/aWDYkW4Us6EuzrcSGbs= github.com/ipfs/go-dsqueue v0.1.2/go.mod h1:OU94YuMVUIF/ctR7Ysov9PI4gOa2XjPGN9nd8imSv78= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483 h1:FnQqL92YxPX08/dcqE4cCSqEzwVGSdj2wprWHX+cUtM= -github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203151407-4b3827ebb483/go.mod h1:YmhRbpaLKg40i9Ogj2+L41tJ+8x50fF8u1FJJD/WNhc= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f h1:6bm6WhriotbESJkTWQxVuiBiXfynOTU5oq3j1bJyNo4= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260203174734-192ec9d15c1f/go.mod h1:fTEpjHMV/G8D1heLf59dVdFVi269m+oGuCKCgFEki3I= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= From 6bb4c1a74527f3a64b4cb02f3322c3fa87cdd4d8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 19:37:39 +0100 Subject: [PATCH 21/22] chore: update boxo to 0a22cde9225c includes refactor of maxLinks check in addLinkChild (review feedback). --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index a880716d2..c0b4d188e 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 + github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 446e2a9e4..6c352a063 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c h1:fWVXPiSYSAkAomQMdRAHUWd7k34eGz2M9emk1bAbvXY= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 70e3f313c..4ef612145 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.8.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 + github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 2c2640384..01aebeb93 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c h1:fWVXPiSYSAkAomQMdRAHUWd7k34eGz2M9emk1bAbvXY= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 53a6ea263..2310081ee 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 // indirect + github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 0b0cfab7a..779fb39cb 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860 h1:II3k9nCyypldZan5gbeE2YtuFdG/X9OaVs9kFc4lxFw= -github.com/ipfs/boxo v0.36.1-0.20260203171618-c6829fe26860/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c h1:fWVXPiSYSAkAomQMdRAHUWd7k34eGz2M9emk1bAbvXY= +github.com/ipfs/boxo v0.36.1-0.20260203183317-0a22cde9225c/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= From a018d14efb667cb350084db550564842fe3476bd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 20:12:11 +0100 Subject: [PATCH 22/22] ci: fix helia-interop and improve caching skip '@helia/mfs - should have the same CID after creating a file' test until helia implements IPIP-499 (tracking: https://github.com/ipfs/helia/issues/941) the test fails because kubo now collapses single-block files to raw CIDs while helia explicitly uses reduceSingleLeafToSelf: false changes: - run aegir directly instead of helia-interop binary (binary ignores --grep flags) - cache node_modules keyed by @helia/interop version from npm registry - skip npm install on cache hit (matches ipfs-webui caching pattern) --- .github/workflows/interop.yml | 40 ++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index f78fd40df..0ba225764 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -71,18 +71,42 @@ jobs: name: kubo path: cmd/ipfs - run: chmod +x cmd/ipfs/ipfs - - run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT - id: npm-cache-dir - - uses: actions/cache@v5 - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-${{ github.job }}-helia-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-${{ github.job }}-helia- - run: sudo apt update - run: sudo apt install -y libxkbcommon0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 # dependencies for playwright - - run: npx --package @helia/interop helia-interop + # Cache node_modules based on latest @helia/interop version from npm registry. + # This ensures we always test against the latest release while still benefiting + # from caching when the version hasn't changed. + - name: Get latest @helia/interop version + id: helia-version + run: echo "version=$(npm view @helia/interop version)" >> $GITHUB_OUTPUT + - name: Cache helia-interop node_modules + uses: actions/cache@v5 + id: helia-cache + with: + path: node_modules + key: ${{ runner.os }}-helia-interop-${{ steps.helia-version.outputs.version }} + - name: Install @helia/interop + if: steps.helia-cache.outputs.cache-hit != 'true' + run: npm install @helia/interop + # TODO(IPIP-499): Remove --grep --invert workaround once helia implements IPIP-499 + # Tracking issue: https://github.com/ipfs/helia/issues/941 + # + # PROVISIONAL HACK: Skip '@helia/mfs - should have the same CID after + # creating a file' test due to IPIP-499 changes in kubo. + # + # WHY IT FAILS: The test creates a 5-byte file in MFS on both kubo and helia, + # then compares the root directory CID. With kubo PR #11148, `ipfs files write` + # now produces raw CIDs for single-block files (matching `ipfs add --raw-leaves`), + # while helia uses `reduceSingleLeafToSelf: false` which keeps the dag-pb wrapper. + # Different file CIDs lead to different directory CIDs. + # + # We run aegir directly (instead of helia-interop binary) because only aegir + # supports the --grep/--invert flags needed to exclude specific tests. + - name: Run helia-interop tests (excluding IPIP-499 incompatible test) + run: npx aegir test -t node --bail -- --grep 'should have the same CID after creating a file' --invert env: KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs + working-directory: node_modules/@helia/interop ipfs-webui: needs: [interop-prep] runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}