From f63887ae9674e682e90eab20d09db19bee81f7d4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 25 Sep 2025 22:49:02 +0000 Subject: [PATCH] fix: provide Filestore nodes (#10990) * Filestore: provide Filestore nodes When strategy is set to "all" (the blockstore does all the providing when a block is written), no providing was happening to Filestore blocks that were not written to the underlying blockstore (so, the DAG leaves, as they live in the filesystem directly). This fixes that. * docs: clarify filestore and urlstore fix in changelog both filestore (local file references) and urlstore (HTTP/HTTPS URL references) blocks are now properly provided shortly after initial add --- core/node/storage.go | 4 ++-- docs/changelogs/v0.38.md | 5 +++++ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 8 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/node/storage.go b/core/node/storage.go index 94237c885..e97a0db4a 100644 --- a/core/node/storage.go +++ b/core/node/storage.go @@ -77,11 +77,11 @@ func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockst } // FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support -func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { +func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks, prov DHTProvider) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { gclocker = blockstore.NewGCLocker() // hash security - fstore = filestore.NewFilestore(bb, repo.FileManager()) + fstore = filestore.NewFilestore(bb, repo.FileManager(), prov) gcbs = blockstore.NewGCBlockstore(fstore, gclocker) gcbs = &verifbs.VerifBSGC{GCBlockstore: gcbs} diff --git a/docs/changelogs/v0.38.md b/docs/changelogs/v0.38.md index 7f50eab9b..649013da5 100644 --- a/docs/changelogs/v0.38.md +++ b/docs/changelogs/v0.38.md @@ -17,6 +17,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [๐ŸŽจ Updated WebUI](#-updated-webui) - [๐Ÿ“Œ Pin name improvements](#-pin-name-improvements) - [๐Ÿ› ๏ธ Identity CID size enforcement and `ipfs files write` fixes](#๏ธ-identity-cid-size-enforcement-and-ipfs-files-write-fixes) + - [Fix: Provide Filestore and Urlstore blocks on write](#๏ธ-provide-filestore-and-urlstore-blocks-on-write) - [๐Ÿ“ฆ๏ธ Important dependency updates](#-important-dependency-updates) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -107,6 +108,10 @@ Identity CIDs use [multihash `0x00`](https://github.com/multiformats/multicodec/ This release resolves several long-standing MFS issues: raw nodes now preserve their codec instead of being forced to dag-pb, append operations on raw nodes work correctly by converting to UnixFS when needed, and identity CIDs properly inherit the full CID prefix from parent directories. +#### Provide Filestore and Urlstore blocks on write + +Improvements to the providing system in the last release (provide blocks according to the configured Strategy) left out [Filestore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) and [Urlstore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore) blocks when the "all" strategy was used. They would only be reprovided but not provided on write. This is now fixed, and both Filestore blocks (local file references) and Urlstore blocks (HTTP/HTTPS URL references) will be provided correctly shortly after initial add. + #### MFS directory cache auto-flush The new [`Internal.MFSAutoflushThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalmfsautoflushthreshold) configuration option prevents unbounded memory growth when using `--flush=false` with `ipfs files` commands by automatically flushing directories when their cache exceeds the configured threshold (default: 256 entries). diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index d44a9977e..e13829789 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f + github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.43.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 822bfadfb..484171c37 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -289,8 +289,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f h1:In4WBnF9sysZwHV6bYTdnLtyRMOQTtvW1leZzxP4bbg= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index a1d466549..76f06b5a7 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f + github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 9bf3c29df..c641a432d 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f h1:In4WBnF9sysZwHV6bYTdnLtyRMOQTtvW1leZzxP4bbg= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index ce363f5cf..8330249e8 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f // indirect + github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.5.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 06c94c5a3..ae2606950 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -332,8 +332,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f h1:In4WBnF9sysZwHV6bYTdnLtyRMOQTtvW1leZzxP4bbg= -github.com/ipfs/boxo v0.34.1-0.20250919000230-031df82b284f/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7 h1:lMzsaKoUSiMmb7xdBbTG0e2Rm85jGlo7fWO/XRhEEDA= +github.com/ipfs/boxo v0.34.1-0.20250925094323-608486081da7/go.mod h1:uhaF0DGnbgEiXDTmD249jCGbxVkMm6+Ew85q6Uub7lo= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk=