feat: Gateway.DeserializedResponses config flag (#9789)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

config/gateway.go

@@ -1,6 +1,9 @@
 package config
 
-const DefaultInlineDNSLink = false
+const (
+	DefaultInlineDNSLink         = false
+	DefaultDeserializedResponses = true
+)
 
 type GatewaySpec struct {
 	// Paths is explicit list of path prefixes that should be handled by
@@ -25,6 +28,11 @@ type GatewaySpec struct {
 	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
 	// and Origin per CID isolation provided by rules like https://publicsuffix.org
 	InlineDNSLink Flag
+
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// responses. Disabling this option enables a Trustless Gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+	DeserializedResponses Flag
 }
 
 // Gateway contains options for the HTTP gateway server.
@@ -56,6 +64,12 @@ type Gateway struct {
 	// This flag can be overridden per FQDN in PublicGateways.
 	NoDNSLink bool
 
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// requests. Disabling this option enables a Trustless only gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can
+	// be overridden per FQDN in PublicGateways.
+	DeserializedResponses Flag
+
 	// PublicGateways configures behavior of known public gateways.
 	// Each key is a fully qualified domain name (FQDN).
 	PublicGateways map[string]*GatewaySpec

core/corehttp/gateway.go

@@ -28,22 +28,11 @@ import (
 
 func GatewayOption(paths ...string) ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		gwConfig, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
 
-		headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
-		for h, v := range cfg.Gateway.HTTPHeaders {
-			headers[http.CanonicalHeaderKey(h)] = v
-		}
-
-		gateway.AddAccessControlHeaders(headers)
-
-		gwConfig := gateway.Config{
-			Headers: headers,
-		}
-
 		gwAPI, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
@@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption {
 
 func HostnameOption() ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		gwConfig, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
@@ -75,9 +64,8 @@ func HostnameOption() ServeOption {
 			return nil, err
 		}
 
-		publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways)
 		childMux := http.NewServeMux()
-		mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP)
+		mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
 		return childMux, nil
 	}
 }
@@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{
 	"localhost": subdomainGatewaySpec,
 }
 
-func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification {
+func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
-	gws := map[string]*gateway.Specification{}
+	cfg, err := n.Repo.Config()
-
+	if err != nil {
-	// First, implicit defaults such as subdomain gateway on localhost
+		return gateway.Config{}, err
-	for hostname, gw := range defaultKnownGateways {
-		gws[hostname] = gw
 	}
 
-	// Then apply values from Gateway.PublicGateways, if present in the config
+	// Parse configuration headers and add the default Access Control Headers.
-	for hostname, gw := range publicGateways {
+	headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
+	for h, v := range cfg.Gateway.HTTPHeaders {
+		headers[http.CanonicalHeaderKey(h)] = v
+	}
+	gateway.AddAccessControlHeaders(headers)
+
+	// Initialize gateway configuration, with empty PublicGateways, handled after.
+	gwCfg := gateway.Config{
+		Headers:               headers,
+		DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
+		NoDNSLink:             cfg.Gateway.NoDNSLink,
+		PublicGateways:        map[string]*gateway.Specification{},
+	}
+
+	// Add default implicit known gateways, such as subdomain gateway on localhost.
+	for hostname, gw := range defaultKnownGateways {
+		gwCfg.PublicGateways[hostname] = gw
+	}
+
+	// Apply values from cfg.Gateway.PublicGateways if they exist.
+	for hostname, gw := range cfg.Gateway.PublicGateways {
 		if gw == nil {
 			// Remove any implicit defaults, if present. This is useful when one
-			// wants to disable subdomain gateway on localhost etc.
+			// wants to disable subdomain gateway on localhost, etc.
-			delete(gws, hostname)
+			delete(gwCfg.PublicGateways, hostname)
 			continue
 		}
 
-		gws[hostname] = &gateway.Specification{
+		gwCfg.PublicGateways[hostname] = &gateway.Specification{
-			Paths:         gw.Paths,
+			Paths:                 gw.Paths,
-			NoDNSLink:     gw.NoDNSLink,
+			NoDNSLink:             gw.NoDNSLink,
-			UseSubdomains: gw.UseSubdomains,
+			UseSubdomains:         gw.UseSubdomains,
-			InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+			InlineDNSLink:         gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+			DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses),
 		}
 	}
 
-	return gws
+	return gwCfg, nil
 }

core/corehttp/gateway_test.go

@@ -14,6 +14,7 @@ import (
 	core "github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/core/coreapi"
 	repo "github.com/ipfs/kubo/repo"
+	"github.com/stretchr/testify/assert"
 
 	iface "github.com/ipfs/boxo/coreiface"
 	nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
@@ -173,3 +174,42 @@ func TestVersion(t *testing.T) {
 		t.Fatalf("response doesn't contain protocol version:\n%s", s)
 	}
 }
+
+func TestDeserializedResponsesInheritance(t *testing.T) {
+	for _, testCase := range []struct {
+		globalSetting          config.Flag
+		gatewaySetting         config.Flag
+		expectedGatewaySetting bool
+	}{
+		{config.True, config.Default, true},
+		{config.False, config.Default, false},
+		{config.False, config.True, true},
+		{config.True, config.False, false},
+	} {
+		c := config.Config{
+			Identity: config.Identity{
+				PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node
+			},
+			Gateway: config.Gateway{
+				DeserializedResponses: testCase.globalSetting,
+				PublicGateways: map[string]*config.GatewaySpec{
+					"example.com": {
+						DeserializedResponses: testCase.gatewaySetting,
+					},
+				},
+			},
+		}
+		r := &repo.Mock{
+			C: c,
+			D: syncds.MutexWrap(datastore.NewMapDatastore()),
+		}
+		n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
+		assert.NoError(t, err)
+
+		gwCfg, err := getGatewayConfig(n)
+		assert.NoError(t, err)
+
+		assert.Contains(t, gwCfg.PublicGateways, "example.com")
+		assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses)
+	}
+}

docs/changelogs/v0.21.md

@@ -7,6 +7,7 @@
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
   - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
+  - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -29,6 +30,32 @@ enabled.
 With this update, the same level of robustness is applied to peers that lack
 mDNS peers and solely rely on the public DHT.
 
+
+#### `Gateway.DeserializedResponses` config flag
+
+This release introduces the
+[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses)
+configuration flag.
+
+With this flag, one can explicitly configure whether the gateway responds to
+deserialized requests or not. By default, this flag is enabled.
+
+Disabling deserialized responses allows the
+gateway to operate
+as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
+limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
+response types:
+[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw),
+[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car),
+and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record).
+
+With deserialized responses disabled, the Kubo gateway can serve as a block
+backend for other software (like
+[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme),
+[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md)
+etc) without the usual risks associated with hosting deserialized data behind
+third-party CIDs.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -50,6 +50,7 @@ config file at runtime.
   - [`Gateway`](#gateway)
     - [`Gateway.NoFetch`](#gatewaynofetch)
     - [`Gateway.NoDNSLink`](#gatewaynodnslink)
+    - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses)
     - [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
     - [`Gateway.RootRedirect`](#gatewayrootredirect)
     - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
@@ -60,6 +61,7 @@ config file at runtime.
       - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains)
       - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink)
       - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink)
+      - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses)
       - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways)
     - [`Gateway` recipes](#gateway-recipes)
   - [`Identity`](#identity)
@@ -236,7 +238,7 @@ documented in `ipfs config profile --help`.
     smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in
     your IPFS node, and disk usage is more critical than performance, consider using
     `flatfs`.
-  - This datastore uses up to several gigabytes of memory.  
+  - This datastore uses up to several gigabytes of memory.
   - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
   - The current implementation is based on old badger 1.x which is no longer supported by the upstream team.
 
@@ -646,6 +648,16 @@ Default: `false`
 
 Type: `bool`
 
+#### `Gateway.DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: `true`
+
+Type: `flag`
+
 ### `Gateway.HTTPHeaders`
 
 Headers to set on gateway responses.
@@ -790,6 +802,16 @@ Default: `false`
 
 Type: `flag`
 
+#### `Gateway.PublicGateways: DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: same as global `Gateway.DeserializedResponses`
+
+Type: `flag`
+
 #### Implicit defaults of `Gateway.PublicGateways`
 
 Default entries for `localhost` hostname and loopback IPs are always present.
@@ -895,7 +917,7 @@ Type: `string` (base64 encoded)
 
 ## `Internal`
 
-This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.  
+This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.
 
 **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.**
 
@@ -971,7 +993,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB)
 ### `Internal.Bitswap.ProviderSearchDelay`
 
 This parameter determines how long to wait before looking for providers outside of bitswap.
-Other routing systems like the DHT are able to provide results in less than a second, so lowering 
+Other routing systems like the DHT are able to provide results in less than a second, so lowering
 this number will allow faster peers lookups in some cases.
 
 Type: `optionalDuration` (`null` means default which is 1s)
@@ -1552,7 +1574,7 @@ another node, even if this other node is on a different network. This may
 trigger netscan alerts on some hosting providers or cause strain in some setups.
 
 The `server` configuration profile fills up this list with sensible defaults,
-preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, 
+preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`,
 which is the multiaddress representation of `192.168.0.0/16`) but you should always
 check settings against your own network and/or hosting provider.
 

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.18
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
+	github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.27.3
 	github.com/multiformats/go-multiaddr v0.9.0

docs/examples/kubo-as-a-library/go.sum

@@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
+	github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

test/cli/gateway_test.go

@@ -513,4 +513,87 @@ func TestGateway(t *testing.T) {
 			})
 		})
 	})
+
+	t.Run("DeserializedResponses", func(t *testing.T) {
+		type testCase struct {
+			globalValue                   config.Flag
+			gatewayValue                  config.Flag
+			deserializedGlobalStatusCode  int
+			deserializedGatewayStaticCode int
+			message                       string
+		}
+
+		setHost := func(r *http.Request) {
+			r.Host = "example.com"
+		}
+
+		withAccept := func(accept string) func(r *http.Request) {
+			return func(r *http.Request) {
+				r.Header.Set("Accept", accept)
+			}
+		}
+
+		withHostAndAccept := func(accept string) func(r *http.Request) {
+			return func(r *http.Request) {
+				setHost(r)
+				withAccept(accept)(r)
+			}
+		}
+
+		makeTest := func(test *testCase) func(t *testing.T) {
+			return func(t *testing.T) {
+				t.Parallel()
+
+				node := harness.NewT(t).NewNode().Init()
+				node.UpdateConfig(func(cfg *config.Config) {
+					cfg.Gateway.DeserializedResponses = test.globalValue
+					cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{
+						"example.com": {
+							Paths:                 []string{"/ipfs", "/ipns"},
+							DeserializedResponses: test.gatewayValue,
+						},
+					}
+				})
+				node.StartDaemon()
+
+				cidFoo := node.IPFSAddStr("foo")
+				client := node.GatewayClient()
+
+				deserializedPath := "/ipfs/" + cidFoo
+
+				blockPath := deserializedPath + "?format=raw"
+				carPath := deserializedPath + "?format=car"
+
+				// Global Check (Gateway.DeserializedResponses)
+				assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode)
+
+				assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode)
+
+				assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode)
+				assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode)
+
+				// Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses)
+				assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode)
+
+				assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode)
+
+				assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode)
+				assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode)
+
+			}
+		}
+
+		for _, test := range []*testCase{
+			{config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"},
+			{config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"},
+			{config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"},
+			{config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"},
+		} {
+			t.Run(test.message, makeTest(test))
+		}
+	})
 }