From 153c870e8d4e257fbb86b27fdd34bb623cd68df2 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:26:17 -1000 Subject: [PATCH 1/3] datastore: upgrade go-ds-flatfs Upgrade go-ds-flatfs to version that uses uses temproary files to store items added to batches. See: https://github.com/ipfs/go-ds-flatfs/pull/142 --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 53fd29136..9abd1ff38 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -79,7 +79,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.5 // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.7 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 7de462937..c5c4bace6 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -314,8 +314,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.5 h1:lkx5C99pFBMI7T1sYF7y3v7xIYekNVNMp/95Gm9Y3tY= -github.com/ipfs/go-ds-flatfs v0.5.5/go.mod h1:bM7+m7KFUyv5dp3RBKTr3+OHgZ6h8ydCQkO7tjeO9Z4= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index 8be0de334..342c16870 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.5 + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.7 diff --git a/go.sum b/go.sum index 1cb7a2d6c..c06c6a85c 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.5 h1:lkx5C99pFBMI7T1sYF7y3v7xIYekNVNMp/95Gm9Y3tY= -github.com/ipfs/go-ds-flatfs v0.5.5/go.mod h1:bM7+m7KFUyv5dp3RBKTr3+OHgZ6h8ydCQkO7tjeO9Z4= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= From 4c805840117f25fdb34564693e7694cd147f9516 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 9 Dec 2025 00:32:47 +0100 Subject: [PATCH 2/3] docs(changelog): add go-ds-flatfs atomic batch writes documents the new flatfs batch implementation that uses atomic operations via temp directory, preventing orphan blocks on interrupted imports and reducing memory usage. --- docs/changelogs/v0.40.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 29780937f..0cf210c46 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -10,8 +10,9 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Atomic batch writes in flatfs datastore](#atomic-batch-writes-in-flatfs-datastore) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - - [Track total size when adding pins](#track-total-size-when-adding-pins] + - [Track total size when adding pins](#track-total-size-when-adding-pins) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -19,6 +20,16 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. ### ๐Ÿ”ฆ Highlights +#### Atomic batch writes in flatfs datastore + +The flatfs datastore batch implementation has been rewritten to use atomic operations. Blocks are now written to a temporary directory first and renamed to their final locations only on commit. + +If kubo is interrupted during `ipfs add` or `ipfs dag import`, uncommitted blocks are automatically cleaned up on the next start, preventing orphan blocks from polluting the datastore. Previously, interrupted imports could leave partial data behind that was difficult to identify and clean up. + +This also reduces memory usage during batch operations, as blocks are written to disk asynchronously instead of being buffered in RAM. + +See [go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142) for implementation details. + #### Routing V1 HTTP API now exposed by default The [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) is now exposed by default at `http://127.0.0.1:8080/routing/v1`. This allows light clients in browsers to use Kubo Gateway as a delegated routing backend instead of running a full DHT client. Support for [IPIP-476: Delegated Routing DHT Closest Peers API](https://github.com/ipfs/specs/pull/476) is included. Can be disabled via [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi). From f422b12ead3ebea34b6f2be9d87e11eee2afb9c5 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Thu, 11 Dec 2025 08:54:25 -1000 Subject: [PATCH 3/3] update go-ds-flatfs --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9abd1ff38..e81cbafd9 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -79,7 +79,7 @@ require ( github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 // indirect + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.7 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c5c4bace6..a0173c76f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -314,8 +314,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= diff --git a/go.mod b/go.mod index 342c16870..9c6cc073b 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-datastore v0.9.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 - github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 + github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.7 diff --git a/go.sum b/go.sum index c06c6a85c..fa515ba3a 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46 h1:B6k+AJpvAM3n+18KSMETdXRoc4z9g5+taVFTudANU34= -github.com/ipfs/go-ds-flatfs v0.5.6-0.20251114193714-784ae9fcab46/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626 h1:F5lZ6Oj6gTRWNkhjIJsixv1j2xemJk0rqvVYKVtOUqE= +github.com/ipfs/go-ds-flatfs v0.5.6-0.20251210234251-f9f889264626/go.mod h1:TtwE6xpVzRMpWDipS1FHEJYxB7YjGNEFZxUXOhTSGGk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo=