From 153c870e8d4e257fbb86b27fdd34bb623cd68df2 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:26:17 -1000 Subject: [PATCH 1/6] datastore: upgrade go-ds-flatfs Upgrade go-ds-flatfs to version that uses uses temproary files to store items added to batches. See: https://github.com/ipfs/go-ds-flatfs/pull/142 --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 53fd29136..9abd1ff38 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -79,7 +79,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.5 // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.7 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 7de462937..c5c4bace6 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -314,8 +314,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.5 h1:lkx5C99pFBMI7T1sYF7y3v7xIYekNVNMp/95Gm9Y3tY= -github.com/ipfs/go-ds-flatfs v0.5.5/go.mod h1:bM7+m7KFUyv5dp3RBKTr3+OHgZ6h8ydCQkO7tjeO9Z4= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index 8be0de334..342c16870 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.5 + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.7 diff --git a/go.sum b/go.sum index 1cb7a2d6c..c06c6a85c 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.5 h1:lkx5C99pFBMI7T1sYF7y3v7xIYekNVNMp/95Gm9Y3tY= -github.com/ipfs/go-ds-flatfs v0.5.5/go.mod h1:bM7+m7KFUyv5dp3RBKTr3+OHgZ6h8ydCQkO7tjeO9Z4= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= From 4c805840117f25fdb34564693e7694cd147f9516 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 9 Dec 2025 00:32:47 +0100 Subject: [PATCH 2/6] docs(changelog): add go-ds-flatfs atomic batch writes documents the new flatfs batch implementation that uses atomic operations via temp directory, preventing orphan blocks on interrupted imports and reducing memory usage. --- docs/changelogs/v0.40.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 29780937f..0cf210c46 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -10,8 +10,9 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Atomic batch writes in flatfs datastore](#atomic-batch-writes-in-flatfs-datastore) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - - [Track total size when adding pins](#track-total-size-when-adding-pins] + - [Track total size when adding pins](#track-total-size-when-adding-pins) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -19,6 +20,16 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. ### ๐Ÿ”ฆ Highlights +#### Atomic batch writes in flatfs datastore + +The flatfs datastore batch implementation has been rewritten to use atomic operations. Blocks are now written to a temporary directory first and renamed to their final locations only on commit. + +If kubo is interrupted during `ipfs add` or `ipfs dag import`, uncommitted blocks are automatically cleaned up on the next start, preventing orphan blocks from polluting the datastore. Previously, interrupted imports could leave partial data behind that was difficult to identify and clean up. + +This also reduces memory usage during batch operations, as blocks are written to disk asynchronously instead of being buffered in RAM. + +See [go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142) for implementation details. + #### Routing V1 HTTP API now exposed by default The [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) is now exposed by default at `http://127.0.0.1:8080/routing/v1`. This allows light clients in browsers to use Kubo Gateway as a delegated routing backend instead of running a full DHT client. Support for [IPIP-476: Delegated Routing DHT Closest Peers API](https://github.com/ipfs/specs/pull/476) is included. Can be disabled via [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi). From f422b12ead3ebea34b6f2be9d87e11eee2afb9c5 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Thu, 11 Dec 2025 08:54:25 -1000 Subject: [PATCH 3/6] update go-ds-flatfs --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9abd1ff38..e81cbafd9 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -79,7 +79,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.7 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c5c4bace6..a0173c76f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -314,8 +314,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index 342c16870..9c6cc073b 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.7 diff --git a/go.sum b/go.sum index c06c6a85c..fa515ba3a 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= From ef0e69bac6ef8034a777217042f977461594ae84 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 12 Jan 2026 22:53:38 +0100 Subject: [PATCH 4/6] chore(deps): update go-ds-flatfs includes improved tests, batch cleanup fixes, and docs --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 68018532d..470fa4dea 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -78,7 +78,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.8 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 118e21cb2..c9f8d0b35 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -288,8 +288,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a h1:+aHKL9O9a8TIx1Ktg1lKYjfIqeRSYz/MmvzsitG5cjQ= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index 527af860d..c16565c35 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 + github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.8 diff --git a/go.sum b/go.sum index fc38d320f..788685934 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a h1:+aHKL9O9a8TIx1Ktg1lKYjfIqeRSYz/MmvzsitG5cjQ= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= From 3d65e8a110358af3a19b7ea72af781042c8ff736 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 13 Jan 2026 01:19:32 +0100 Subject: [PATCH 5/6] docs(changelog): reframe go-ds-flatfs entry for users focus on user benefits instead of implementation details --- docs/changelogs/v0.40.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 147dc0511..adc7f252c 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -10,7 +10,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - - [Atomic batch writes in flatfs datastore](#atomic-batch-writes-in-flatfs-datastore) + - [๐Ÿงน Automatic cleanup of interrupted imports](#-automatic-cleanup-of-interrupted-imports) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - [Track total size when adding pins](#track-total-size-when-adding-pins) - [๐Ÿš‡ Improved `ipfs p2p` tunnels with foreground mode](#-improved-ipfs-p2p-tunnels-with-foreground-mode) @@ -25,15 +25,13 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. ### ๐Ÿ”ฆ Highlights -#### Atomic batch writes in flatfs datastore +#### ๐Ÿงน Automatic cleanup of interrupted imports -The flatfs datastore batch implementation has been rewritten to use atomic operations. Blocks are now written to a temporary directory first and renamed to their final locations only on commit. +If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection. -If kubo is interrupted during `ipfs add` or `ipfs dag import`, uncommitted blocks are automatically cleaned up on the next start, preventing orphan blocks from polluting the datastore. Previously, interrupted imports could leave partial data behind that was difficult to identify and clean up. +Batch operations also use less memory now. Block data is written to disk immediately rather than held in RAM until the batch commits. -This also reduces memory usage during batch operations, as blocks are written to disk asynchronously instead of being buffered in RAM. - -See [go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142) for implementation details. +Under the hood, the block storage layer (flatfs) was rewritten to use atomic batch operations via a temporary staging directory. See [go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142) for details. #### Routing V1 HTTP API now exposed by default From f6eff8df150f29f7e32230514d0d97e82be6c9b4 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Jan 2026 18:41:23 -1000 Subject: [PATCH 6/6] update go-ds-flatfs --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 470fa4dea..3abd1b0fd 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -78,7 +78,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.8 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c9f8d0b35..c623bdbce 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -288,8 +288,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a h1:+aHKL9O9a8TIx1Ktg1lKYjfIqeRSYz/MmvzsitG5cjQ= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408 h1:xw1mz0Y5Z7H2MKRpnOmqpqBBVMr2TT2DlQB+6hZiqpc= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408/go.mod h1:p8a/YhmAFYyuonxDbvuIANlDCgS69uqVv+iH5f8fAxY= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index c16565c35..2c2b0a829 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a + github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.8 diff --git a/go.sum b/go.sum index 788685934..c0546eb7e 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a h1:+aHKL9O9a8TIx1Ktg1lKYjfIqeRSYz/MmvzsitG5cjQ= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20260112192853-bd1c3b5bea8a/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408 h1:xw1mz0Y5Z7H2MKRpnOmqpqBBVMr2TT2DlQB+6hZiqpc= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20260113044005-5929cc55c408/go.mod h1:p8a/YhmAFYyuonxDbvuIANlDCgS69uqVv+iH5f8fAxY= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo=