From c2d414f8fa7fd5d2a39c1ba85ca71d201947576b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Feb 2026 00:30:19 +0100 Subject: [PATCH] fix(mfs): single-block files in CIDv1 dirs now produce raw CIDs problem: `ipfs files write` in CIDv1 directories wrapped single-block files in dag-pb even when raw-leaves was enabled, producing different CIDs than `ipfs add --raw-leaves` for the same content. fix: boxo now collapses single-block ProtoNode wrappers (with no metadata) to RawNode in DagModifier.GetNode(). files with mtime/mode stay as dag-pb since raw blocks cannot store UnixFS metadata. also fixes sparse file writes where writing past EOF would lose data because expandSparse didn't update the internal node pointer. updates boxo to v0.36.1-0.20260203003133-7884ae23aaff updates t0250-files-api.sh test hashes to match new behavior --- docs/changelogs/v0.40.md | 6 ++++- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- test/sharness/t0250-files-api.sh | 31 +++++++++++++++++--------- 8 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index a8271910e..735bf014c 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -59,9 +59,13 @@ The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs - `--hidden` / `-H` includes hidden files (default: false) - `--trickle` implicit default can be adjusted via `Import.UnixFSDAGLayout` +**`ipfs files write` fix for CIDv1 directories** + +When writing to MFS directories that use CIDv1 (via `--cid-version=1` or `ipfs files chcid`), single-block files now produce raw block CIDs (like `bafkrei...`), matching the behavior of `ipfs add --raw-leaves`. Previously, MFS would wrap single-block files in dag-pb even when raw leaves were enabled. CIDv0 directories continue to use dag-pb. + **HAMT Threshold Fix** -HAMT directory sharding threshold changed from `>=` to `>` to match the GO docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. +HAMT directory sharding threshold changed from `>=` to `>` to match the Go docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. #### 🧹 Automatic cleanup of interrupted imports diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 596f7cd83..c6c3ab14b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 0601423b8..cdb00389d 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index dcc21f033..6cc5e5c43 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 6745ce6dc..95e53c165 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 6dc40cd66..c5a2a5af5 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab // indirect + github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 7942c95e4..755148e26 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab h1:zPSrjZIKHEgtR5JVocSgymVpQxUrIXUw3gA+0Clo69M= -github.com/ipfs/boxo v0.36.1-0.20260202024233-ac97424d99ab/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff h1:cfQz8LBzOp3VxEMQG+SUWG4TIvGdfg2rZLvga8uPRnw= +github.com/ipfs/boxo v0.36.1-0.20260203003133-7884ae23aaff/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index b86ee56f5..ad9ca5f81 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -786,6 +786,7 @@ tests_for_files_api() { test_expect_success "can create some files for testing ($EXTRA)" ' create_files ' + # default: CIDv0, dag-pb for all files (no raw-leaves) ROOT_HASH=QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt CATS_HASH=Qma88m8ErTGkZHbBWGqy1C7VmEmX8wwNDWNpGyCaNmEgwC FILE_HASH=QmQdQt9qooenjeaNhiKHF3hBvmNteB4MQBtgu3jxgf9c7i @@ -796,20 +797,23 @@ tests_for_files_api() { create_files --raw-leaves ' + # partial raw-leaves: initial files created with --raw-leaves, test ops without if [ "$EXTRA" = "with-daemon" ]; then ROOT_HASH=QmTpKiKcAj4sbeesN6vrs5w3QeVmd4QmGpxRL81hHut4dZ CATS_HASH=QmPhPkmtUGGi8ySPHoPu1qbfryLJKKq1GYxpgLyyCruvGe test_files_api "($EXTRA, partial raw-leaves)" fi - ROOT_HASH=QmW3dMSU6VNd1mEdpk9S3ZYRuR1YwwoXjGaZhkyK6ru9YU - CATS_HASH=QmPqWDEg7NoWRX8Y4vvYjZtmdg5umbfsTQ9zwNr12JoLmt - FILE_HASH=QmRCgHeoKxCqK2Es6M6nPUDVWz19yNQPnsXGsXeuTkSKpN - TRUNC_HASH=QmckstrVxJuecVD1FHUiURJiU9aPURZWJieeBVHJPACj8L + # raw-leaves: single-block files become RawNode (CIDv1), dirs stay CIDv0 + ROOT_HASH=QmTHzLiSouBHVTssS8xRzmfWGAvTGhPEjtPdB6pWMQdxJX + CATS_HASH=QmPJkzbCoBuL379TbHgwF1YbVHnKgiDa5bjqYhe6Lovdms + FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm + TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e test_files_api "($EXTRA, raw-leaves)" '' --raw-leaves - ROOT_HASH=QmageRWxC7wWjPv5p36NeAgBAiFdBHaNfxAehBSwzNech2 - CATS_HASH=bafybeig4cpvfu2qwwo3u4ffazhqdhyynfhnxqkzvbhrdbamauthf5mfpuq + # cidv1 for mkdir: different from raw-leaves since mkdir forces CIDv1 dirs + ROOT_HASH=QmTLdTaZNj8Mvq1cgYup59ZFJFv1KxptouFSZUZKeq7X3z + CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e if [ "$EXTRA" = "with-daemon" ]; then @@ -823,8 +827,10 @@ tests_for_files_api() { test_cmp hash_expect hash_actual ' - ROOT_HASH=bafybeifxnoetaa2jetwmxubv3gqiyaknnujwkkkhdeua63kulm63dcr5wu - test_files_api "($EXTRA, cidv1 root)" + # cidv1 root: root upgraded to CIDv1 via chcid, all new dirs/files also CIDv1 + ROOT_HASH=bafybeickjecu37qv6ue54ofk3n4rpm4g4abuofz7yc4qn4skffy263kkou + CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm + test_files_api "($EXTRA, cidv1 root)" if [ "$EXTRA" = "with-daemon" ]; then test_expect_success "can update root hash to blake2b-256" ' @@ -833,8 +839,9 @@ tests_for_files_api() { ipfs files stat --hash / > hash_actual && test_cmp hash_expect hash_actual ' - ROOT_HASH=bafykbzaceb6jv27itwfun6wsrbaxahpqthh5be2bllsjtb3qpmly3vji4mlfk - CATS_HASH=bafykbzacebhpn7rtcjjc5oa4zgzivhs7a6e2tq4uk4px42bubnmhpndhqtjig + # blake2b-256 root: using blake2b-256 hash instead of sha2-256 + ROOT_HASH=bafykbzaceaebvwrjdw5rfhqqh5miaq3g42yybnrw3kxxxx43ggyttm6xn2zek + CATS_HASH=bafykbzaceaqvpxs3dfl7su6744jgyvifbusow2tfixdy646chasdwyz2boagc FILE_HASH=bafykbzaceca45w2i3o3q3ctqsezdv5koakz7sxsw37ygqjg4w54m2bshzevxy TRUNC_HASH=bafykbzaceadeu7onzmlq7v33ytjpmo37rsqk2q6mzeqf5at55j32zxbcdbwig test_files_api "($EXTRA, blake2b-256 root)" @@ -866,10 +873,12 @@ test_expect_success "enable sharding in config" ' test_launch_ipfs_daemon_without_network +# sharding cidv0: HAMT-sharded directory with 100 files, CIDv0 SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se test_sharding "(cidv0)" -SHARD_HASH=bafybeib46tpawg2d2hhlmmn2jvgio33wqkhlehxrem7wbfvqqikure37rm +# sharding cidv1: HAMT-sharded directory with 100 files, CIDv1 +SHARD_HASH=bafybeiaulcf7c46pqg3tkud6dsvbgvlnlhjuswcwtfhxts5c2kuvmh5keu test_sharding "(cidv1 root)" "--cid-version=1" test_kill_ipfs_daemon