From 60e56681e3331ba1408bf74b18e81cc2586f1278 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 3 Dec 2025 05:08:01 +0100 Subject: [PATCH] feat(gateway): IPIP-0524 Gateway.AllowCodecConversion config option Wire up boxo's AllowCodecConversion config to control codec conversion behavior per IPIP-0524. When false (default), the gateway returns 406 Not Acceptable if the requested format doesn't match the block's codec. Clients should fetch raw blocks (`?format=raw`) and convert client-side. Ref: https://github.com/ipfs/specs/pull/524 Ref: https://github.com/ipfs/boxo/pull/1077 Ref: https://github.com/ipfs/gateway-conformance/pull/254 --- .github/workflows/gateway-conformance.yml | 8 ++++---- config/gateway.go | 7 +++++++ core/corehttp/gateway.go | 1 + docs/changelogs/v0.40.md | 15 ++++++++++++++- docs/config.md | 23 +++++++++++++++++++++++ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 11 files changed, 58 insertions(+), 14 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index b62fd601f..0202922d6 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -41,7 +41,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures @@ -93,7 +93,7 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8080 subdomain-url: http://localhost:8080 @@ -127,7 +127,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures @@ -199,7 +199,7 @@ jobs: # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8092 args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' diff --git a/config/gateway.go b/config/gateway.go index 3495caede..c535d839f 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -10,6 +10,7 @@ const ( DefaultDisableHTMLErrors = false DefaultExposeRoutingAPI = true DefaultDiagnosticServiceURL = "https://check.ipfs.network" + DefaultAllowCodecConversion = false // Gateway limit defaults from boxo DefaultRetrievalTimeout = gateway.DefaultRetrievalTimeout @@ -72,6 +73,12 @@ type Gateway struct { // be overridden per FQDN in PublicGateways. DeserializedResponses Flag + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec (e.g., + // converting dag-pb or dag-cbor to dag-json). When disabled, the gateway + // returns 406 Not Acceptable for codec mismatches per IPIP-0524. + AllowCodecConversion Flag + // DisableHTMLErrors disables pretty HTML pages when an error occurs. Instead, a `text/plain` // page will be sent with the raw error message. DisableHTMLErrors Flag diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 393a668bf..84ad0384c 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -268,6 +268,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, map[string][]string, er // Initialize gateway configuration, with empty PublicGateways, handled after. gwCfg := gateway.Config{ DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), + AllowCodecConversion: cfg.Gateway.AllowCodecConversion.WithDefault(config.DefaultAllowCodecConversion), DisableHTMLErrors: cfg.Gateway.DisableHTMLErrors.WithDefault(config.DefaultDisableHTMLErrors), NoDNSLink: cfg.Gateway.NoDNSLink, PublicGateways: map[string]*gateway.PublicGateway{}, diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 29780937f..de11a3131 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -11,7 +11,8 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - - [Track total size when adding pins](#track-total-size-when-adding-pins] + - [Track total size when adding pins](#track-total-size-when-adding-pins) + - [IPIP-0524: Gateway codec conversion disabled by default](#ipip-0524-gateway-codec-conversion-disabled-by-default) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -32,6 +33,18 @@ Example output: Fetched/Processed 336 nodes (83 MB) ``` +#### IPIP-0524: Gateway codec conversion disabled by default + +Codec conversion is now disabled by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). +Requests for a format that differs from the block's codec will return `406 Not Acceptable`. + +**Migration**: Clients should fetch raw blocks (`?format=raw` or `Accept: application/vnd.ipld.raw`) +and convert client-side using libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch). + +This change removes gateways from a gatekeeping role: new codecs can now be adopted by clients +immediately without waiting for gateway operator updates. Set [`Gateway.AllowCodecConversion`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayallowcodecconversion) +to `true` to restore previous behavior. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index 23386f7e6..92f2e81a2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -63,6 +63,7 @@ config file at runtime. - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses) + - [`Gateway.AllowCodecConversion`](#gatewayallowcodecconversion) - [`Gateway.DisableHTMLErrors`](#gatewaydisablehtmlerrors) - [`Gateway.ExposeRoutingAPI`](#gatewayexposeroutingapi) - [`Gateway.RetrievalTimeout`](#gatewayretrievaltimeout) @@ -1104,6 +1105,28 @@ Default: `true` Type: `flag` +### `Gateway.AllowCodecConversion` + +An optional flag to enable automatic conversion between codecs when the +requested format differs from the block's native codec (e.g., converting +dag-pb or dag-cbor to dag-json). + +When disabled (the default), the gateway returns `406 Not Acceptable` for +codec mismatches, following behavior specified in +[IPIP-0524](https://github.com/ipfs/specs/pull/524). + +**Recommended approach**: Instead of relying on gateway-side conversion, +fetch the raw block using `?format=raw` (`application/vnd.ipld.raw`) and +perform conversion client-side. This approach: + +- Allows clients to use any codec without waiting for gateway support +- Enables ecosystem innovation without gateway operator coordination +- Works with libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch) in JavaScript + +Default: `false` + +Type: `flag` + ### `Gateway.DisableHTMLErrors` An optional flag to disable the pretty HTML error pages of the gateway. Instead, diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 53fd29136..7ff53bbef 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c + github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.45.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 7de462937..6e69125c4 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -291,8 +291,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c h1:mczpALnNzNhmggehO5Ehr9+Q8+NiJyKJfT4EPwi01d0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f h1:QaTdH/7Bi3wPYtwRs9b7cKguT2h2al1ZLYE+ziFYqEg= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index 8be0de334..f3b6f58f8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c + github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 1cb7a2d6c..8dc5f381f 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c h1:mczpALnNzNhmggehO5Ehr9+Q8+NiJyKJfT4EPwi01d0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f h1:QaTdH/7Bi3wPYtwRs9b7cKguT2h2al1ZLYE+ziFYqEg= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index f9121746e..e89a918b5 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -136,7 +136,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c // indirect + github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index c5fba6ef6..5eb832463 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -334,8 +334,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c h1:mczpALnNzNhmggehO5Ehr9+Q8+NiJyKJfT4EPwi01d0= -github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f h1:QaTdH/7Bi3wPYtwRs9b7cKguT2h2al1ZLYE+ziFYqEg= +github.com/ipfs/boxo v0.35.3-0.20251203042105-52a05615b40f/go.mod h1:Abmp1if6bMQG87/0SQPIB9fkxJnZMLCt2nQw3yUZHH0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk=