From d6069b93ee8a2f23ca478fbc5938b5e9231a0f22 Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Tue, 3 Jan 2023 21:10:33 +0100 Subject: [PATCH] fix: disable provide over HTTP with Routing.Type=auto (#9511) Closes https://github.com/ipfs/kubo/issues/9504 --- core/node/libp2p/routingopt.go | 20 ++++++- docs/environment-variables.md | 17 ++++++ docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 +-- go.mod | 4 +- go.sum | 8 +-- routing/composer.go | 9 ++++ .../t0172-content-routing-over-http.sh | 54 +++++++++++++++++++ 8 files changed, 111 insertions(+), 13 deletions(-) create mode 100755 test/sharness/t0172-content-routing-over-http.sh diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 2d2b7570c..bfb45971c 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -2,6 +2,8 @@ package libp2p import ( "context" + "os" + "strings" "time" "github.com/ipfs/go-datastore" @@ -30,6 +32,13 @@ var defaultHTTPRouters = []string{ // TODO: add an independent router from Cloudflare } +func init() { + // Override HTTP routers if custom ones were passed via env + if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" { + defaultHTTPRouters = strings.Split(routers, " ") + } +} + // ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto" func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func( ctx context.Context, @@ -67,8 +76,17 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func if err != nil { return nil, err } + + r := &irouting.Composer{ + GetValueRouter: routinghelpers.Null{}, + PutValueRouter: routinghelpers.Null{}, + ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide + FindPeersRouter: routinghelpers.Null{}, + FindProvidersRouter: httpRouter, + } + routers = append(routers, &routinghelpers.ParallelRouter{ - Router: httpRouter, + Router: r, IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387 Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529 ExecuteAfter: 0, diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 98b449054..add8592a0 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -114,6 +114,23 @@ $ ipfs resolve -r /ipns/dnslink-test2.example.com /ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am ``` +## `IPFS_HTTP_ROUTERS` + +Overrides all implicit HTTP routers enabled when `Routing.Type=auto` with +the space-separated list of URLs provided in this variable. +Useful for testing and debugging in offline contexts. + +Example: + +```console +$ ipfs config Routing.Type auto +$ IPFS_HTTP_ROUTERS="http://127.0.0.1:7423" ipfs daemon +``` + +The above will replace implicit HTTP routers with single one, allowing for +inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done` +or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy). + ## `LIBP2P_TCP_REUSEPORT` Kubo tries to reuse the same source port for all connections to improve NAT diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 1c40cfe81..1846a1ea9 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -127,7 +127,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.4.1 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.5.0 // indirect github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-mplex v0.7.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect @@ -198,7 +198,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/crypto v0.3.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f2d9d7ebd..fa464c50e 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -793,8 +793,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.4.1 h1:rOlZiFpUt7SgHm4w62MBvWaQ4UHh7bVJnSnor6RN7j8= -github.com/libp2p/go-libp2p-routing-helpers v0.4.1/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= +github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4= +github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -1377,8 +1377,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= diff --git a/go.mod b/go.mod index f214d5743..bb0dd2930 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.8.2 github.com/libp2p/go-libp2p-pubsub-router v0.6.0 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.4.0 + github.com/libp2p/go-libp2p-routing-helpers v0.5.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-socket-activation v0.1.0 github.com/miekg/dns v1.1.50 @@ -231,7 +231,7 @@ require ( go.opentelemetry.io/otel/metric v0.30.0 // indirect go.opentelemetry.io/proto/otlp v0.16.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.9.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/net v0.3.0 // indirect diff --git a/go.sum b/go.sum index f890f22b0..435c88bc2 100644 --- a/go.sum +++ b/go.sum @@ -828,8 +828,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= -github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= +github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4= +github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -1441,8 +1441,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= diff --git a/routing/composer.go b/routing/composer.go index f2f1f65e6..f54d954bd 100644 --- a/routing/composer.go +++ b/routing/composer.go @@ -100,9 +100,18 @@ func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Opt func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { log.Debug("composer: calling searchValue: ", key) ch, err := c.GetValueRouter.SearchValue(ctx, key, opts...) + + // avoid nil channels on implementations not supporting SearchValue method. + if err == routing.ErrNotFound && ch == nil { + out := make(chan []byte) + close(out) + return out, err + } + if err != nil { log.Debug("composer: calling searchValue error: ", key, err) } + return ch, err } diff --git a/test/sharness/t0172-content-routing-over-http.sh b/test/sharness/t0172-content-routing-over-http.sh new file mode 100755 index 000000000..bc35c6328 --- /dev/null +++ b/test/sharness/t0172-content-routing-over-http.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +test_description="Test content routing over HTTP" + +. lib/test-lib.sh + + +if ! test_have_prereq SOCAT; then + skip_all="skipping '$test_description': socat is not available" + test_done +fi + +test_init_ipfs + +# Run listener on a free port to log HTTP requests sent by Kubo in Routing.Type=auto mode +export ROUTER_PORT=$(comm -23 <(seq 49152 65535 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2) | head -n 1) +export IPFS_HTTP_ROUTERS="http://127.0.0.1:$ROUTER_PORT" + +test_launch_ipfs_daemon + +test_expect_success "start HTTP router proxy" ' + socat TCP-LISTEN:$ROUTER_PORT,reuseaddr,fork,bind=127.0.0.1 STDOUT > http_requests & + NCPID=$! +' + +## HTTP GETs + +test_expect_success 'create unique CID without adding it to the local datastore' ' + WANT_CID=$(date +"%FT%T.%N%z" | ipfs add -qn) +' + +test_expect_success 'expect HTTP request for unknown CID' ' + ipfs routing findprovs --timeout 3s "$WANT_CID" && + test_should_contain "GET /routing/v1/providers/$WANT_CID" http_requests +' + +## HTTP PUTs + +test_expect_success 'add new CID to the local datastore' ' + ADD_CID=$(date +"%FT%T.%N%z" | ipfs add -q) +' + +# cid.contact supports GET-only: https://github.com/ipfs/kubo/issues/9504 +# which means no announcements over HTTP should be made. +test_expect_success 'expect no HTTP requests to be sent with locally added CID' ' + test_should_not_contain "$ADD_CID" http_requests +' + +test_expect_success "stop nc" ' + kill "$NCPID" && wait "$NCPID" || true +' + +test_kill_ipfs_daemon +test_done