mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
feat: opt-in http retrieval client (#10772)
* Feat: http retrieval as experimental feature This introduces the http-retrieval capability as an experimental feature. It can be enabled in the configuration `Experimental.HTTPRetrieval.Enabled = true`. Documentation and changelog to be added later. * refactor: HTTPRetrieval.Enabled as Flag * docs(config): HTTPRetrieval section * refactor: reusable MockHTTPContentRouter * feat: HTTPRetrieval.TLSInsecureSkipVerify allows self-signed certificates in tests * feat(config): HTTPRetrieval.MaxBlockSize * test: end-to-end HTTPRetrieval.Enabled this spawns two http services on localhost: 1. HTTP router that returns HTTP provider when /routing/v1/providers/cid i queried 2. HTTP provider that returns a block when /ipfs/cid is queried 3. Configures Kubo to use (1) instead of cid.contact this seems to work (running test with DEBUG=true shows (1) was queried for the test CID and returned multiaddr of (2), but Kubo never requested test CID block from (2) – needs investigation * fix: enable /routing/v1/peers for non-cid.contact we artificially limited every delegated routing endpoint because of cid.contact being limited to one endpoint * feat: Routing.DelegatedRouters make it easy to override the hardcoded implicit HTTP routeur URL without having to set the entire custom Router.Routers and Router.Methods (http_retrieval_client_test.go still needs to be fixed in future commit) * test: flag remaining work * docs: review feedback * refactor: providerQueryMgr with bitswapNetworks this fixes two regressions: (1) introduced in https://github.com/ipfs/kubo/issues/10717 where we only used bitswapLib2p query manager (this is why E2E did not act on http provider) (2) introduced in https://github.com/ipfs/kubo/pull/10765 where it was not possible to set binary peerID in IgnoreProviders (we changed to []string) * refactor: Bitswap.Libp2pEnabled replaces Bitswap.Enabled with Bitswap.Libp2pEnabled adds tests that confirm it is possible to disable libp2p bitswap fully and only keep http in client mode also, removes the need for passing empty blockstore in client-only mode * docs: changelog --------- Co-authored-by: Marcin Rataj <lidel@lidel.org>
This commit is contained in:
Hector Sanjuan
GitHub
parent
7059620181
commit
b5d73695ba
@ -458,6 +458,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
|
||||
cfg.Identity.PeerID,
|
||||
cfg.Addresses,
|
||||
cfg.Identity.PrivKey,
|
||||
cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled),
|
||||
)
|
||||
default:
|
||||
return fmt.Errorf("unrecognized routing option: %s", routingOption)
|
||||
|
||||
@ -2,13 +2,14 @@ package config
|
||||
|
||||
// Bitswap holds Bitswap configuration options
|
||||
type Bitswap struct {
|
||||
// Enabled controls both client and server (enabled by default)
|
||||
Enabled Flag `json:",omitempty"`
|
||||
// ServerEnabled controls if the node responds to WANTs (depends on Enabled, enabled by default)
|
||||
// Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default)
|
||||
// (This can be disabled if HTTPRetrieval.Enabled is set to true)
|
||||
Libp2pEnabled Flag `json:",omitempty"`
|
||||
// ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default)
|
||||
ServerEnabled Flag `json:",omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultBitswapEnabled = true
|
||||
DefaultBitswapLibp2pEnabled = true
|
||||
DefaultBitswapServerEnabled = true
|
||||
)
|
||||
|
||||
@ -33,13 +33,14 @@ type Config struct {
|
||||
DNS DNS
|
||||
Migration Migration
|
||||
|
||||
Provider Provider
|
||||
Reprovider Reprovider
|
||||
Experimental Experiments
|
||||
Plugins Plugins
|
||||
Pinning Pinning
|
||||
Import Import
|
||||
Version Version
|
||||
Provider Provider
|
||||
Reprovider Reprovider
|
||||
HTTPRetrieval HTTPRetrieval
|
||||
Experimental Experiments
|
||||
Plugins Plugins
|
||||
Pinning Pinning
|
||||
Import Import
|
||||
Version Version
|
||||
|
||||
Internal Internal // experimental/unstable options
|
||||
|
||||
|
||||
19
config/http_retrieval.go
Normal file
19
config/http_retrieval.go
Normal file
@ -0,0 +1,19 @@
|
||||
package config
|
||||
|
||||
// HTTPRetrieval is the configuration object for HTTP Retrieval settings.
|
||||
// Implicit defaults can be found in core/node/bitswap.go
|
||||
type HTTPRetrieval struct {
|
||||
Enabled Flag `json:",omitempty"`
|
||||
Allowlist []string `json:",omitempty"`
|
||||
Denylist []string `json:",omitempty"`
|
||||
NumWorkers *OptionalInteger `json:",omitempty"`
|
||||
MaxBlockSize *OptionalString `json:",omitempty"`
|
||||
TLSInsecureSkipVerify Flag `json:",omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPRetrievalEnabled = false // opt-in for now, until we figure out https://github.com/ipfs/specs/issues/496
|
||||
DefaultHTTPRetrievalNumWorkers = 16
|
||||
DefaultHTTPRetrievalTLSInsecureSkipVerify = false // only for testing with self-signed HTTPS certs
|
||||
DefaultHTTPRetrievalMaxBlockSize = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes
|
||||
)
|
||||
@ -48,13 +48,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
|
||||
},
|
||||
},
|
||||
|
||||
Routing: Routing{
|
||||
Type: nil,
|
||||
Methods: nil,
|
||||
Routers: nil,
|
||||
IgnoreProviders: []peer.ID{},
|
||||
},
|
||||
|
||||
// setup the node mount points.
|
||||
Mounts: Mounts{
|
||||
IPFS: "/ipfs",
|
||||
|
||||
@ -6,27 +6,30 @@ import (
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
peer "github.com/libp2p/go-libp2p/core/peer"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultAcceleratedDHTClient = false
|
||||
DefaultLoopbackAddressesOnLanDHT = false
|
||||
CidContactRoutingURL = "https://cid.contact"
|
||||
PublicGoodDelegatedRoutingURL = "https://delegated-ipfs.dev" // cid.contact + amino dht (incl. IPNS PUTs)
|
||||
EnvHTTPRouters = "IPFS_HTTP_ROUTERS"
|
||||
EnvHTTPRoutersFilterProtocols = "IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS"
|
||||
)
|
||||
|
||||
var (
|
||||
// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
|
||||
DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{
|
||||
"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
|
||||
DefaultHTTPRouters = getEnvOrDefault(EnvHTTPRouters, []string{
|
||||
CidContactRoutingURL, // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
|
||||
})
|
||||
|
||||
// Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484)
|
||||
// and also filter out locally
|
||||
DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{
|
||||
DefaultHTTPRoutersFilterProtocols = getEnvOrDefault(EnvHTTPRoutersFilterProtocols, []string{
|
||||
"unknown", // allow results without protocol list, we can do libp2p identify to test them
|
||||
"transport-bitswap",
|
||||
// TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed
|
||||
// http is added dynamically in routing/delegated.go.
|
||||
// 'transport-ipfs-gateway-http'
|
||||
})
|
||||
)
|
||||
|
||||
@ -43,11 +46,13 @@ type Routing struct {
|
||||
|
||||
LoopbackAddressesOnLanDHT Flag `json:",omitempty"`
|
||||
|
||||
IgnoreProviders []peer.ID
|
||||
IgnoreProviders []string `json:",omitempty"`
|
||||
|
||||
Routers Routers
|
||||
DelegatedRouters []string `json:",omitempty"`
|
||||
|
||||
Methods Methods
|
||||
Routers Routers `json:",omitempty"`
|
||||
|
||||
Methods Methods `json:",omitempty"`
|
||||
}
|
||||
|
||||
type Router struct {
|
||||
|
||||
@ -2,24 +2,28 @@ package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/ipfs/boxo/bitswap"
|
||||
"github.com/ipfs/boxo/bitswap/client"
|
||||
"github.com/ipfs/boxo/bitswap/network"
|
||||
bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
|
||||
"github.com/ipfs/boxo/bitswap/network/httpnet"
|
||||
blockstore "github.com/ipfs/boxo/blockstore"
|
||||
exchange "github.com/ipfs/boxo/exchange"
|
||||
"github.com/ipfs/boxo/exchange/providing"
|
||||
provider "github.com/ipfs/boxo/provider"
|
||||
rpqm "github.com/ipfs/boxo/routing/providerquerymanager"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-datastore"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
version "github.com/ipfs/kubo"
|
||||
"github.com/ipfs/kubo/config"
|
||||
irouting "github.com/ipfs/kubo/routing"
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/routing"
|
||||
peer "github.com/libp2p/go-libp2p/core/peer"
|
||||
"go.uber.org/fx"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
@ -79,38 +83,63 @@ type bitswapIn struct {
|
||||
// Bitswap creates the BitSwap server/client instance.
|
||||
// If Bitswap.ServerEnabled is false, the node will act only as a client
|
||||
// using an empty blockstore to prevent serving blocks to other peers.
|
||||
func Bitswap(provide bool) interface{} {
|
||||
func Bitswap(serverEnabled bool) interface{} {
|
||||
return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
|
||||
bitswapNetwork := bsnet.NewFromIpfsHost(in.Host)
|
||||
var blockstoree blockstore.Blockstore = in.Bs
|
||||
var provider routing.ContentDiscovery
|
||||
var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
|
||||
var bitswapBlockstore blockstore.Blockstore = in.Bs
|
||||
|
||||
if provide {
|
||||
libp2pEnabled := in.Cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
|
||||
if libp2pEnabled {
|
||||
bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
|
||||
}
|
||||
|
||||
var maxProviders int = DefaultMaxProviders
|
||||
if in.Cfg.Internal.Bitswap != nil {
|
||||
maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
|
||||
}
|
||||
|
||||
pqm, err := rpqm.New(bitswapNetwork,
|
||||
in.Rt,
|
||||
rpqm.WithMaxProviders(maxProviders),
|
||||
rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...),
|
||||
)
|
||||
if httpCfg := in.Cfg.HTTPRetrieval; httpCfg.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) {
|
||||
maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
|
||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(true))
|
||||
provider = pqm
|
||||
bitswapHTTP := httpnet.New(in.Host,
|
||||
httpnet.WithHTTPWorkers(int(httpCfg.NumWorkers.WithDefault(config.DefaultHTTPRetrievalNumWorkers))),
|
||||
httpnet.WithAllowlist(httpCfg.Allowlist),
|
||||
httpnet.WithDenylist(httpCfg.Denylist),
|
||||
httpnet.WithInsecureSkipVerify(httpCfg.TLSInsecureSkipVerify.WithDefault(config.DefaultHTTPRetrievalTLSInsecureSkipVerify)),
|
||||
httpnet.WithMaxBlockSize(int64(maxBlockSize)),
|
||||
httpnet.WithUserAgent(version.GetUserAgentVersion()),
|
||||
)
|
||||
bitswapNetworks = network.New(in.Host.Peerstore(), bitswapLibp2p, bitswapHTTP)
|
||||
} else if libp2pEnabled {
|
||||
bitswapNetworks = bitswapLibp2p
|
||||
} else {
|
||||
provider = nil
|
||||
// When server is disabled, use an empty blockstore to prevent serving blocks
|
||||
blockstoree = blockstore.NewBlockstore(datastore.NewMapDatastore())
|
||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(false))
|
||||
return nil, errors.New("invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
|
||||
}
|
||||
|
||||
bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, blockstoree, in.BitswapOpts...)
|
||||
// Kubo uses own, customized ProviderQueryManager
|
||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
|
||||
var maxProviders int = DefaultMaxProviders
|
||||
if in.Cfg.Internal.Bitswap != nil {
|
||||
maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
|
||||
}
|
||||
ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders))
|
||||
for _, str := range in.Cfg.Routing.IgnoreProviders {
|
||||
pid, err := peer.Decode(str)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ignoredPeerIDs = append(ignoredPeerIDs, pid)
|
||||
}
|
||||
providerQueryMgr, err := rpqm.New(bitswapNetworks,
|
||||
in.Rt,
|
||||
rpqm.WithMaxProviders(maxProviders),
|
||||
rpqm.WithIgnoreProviders(ignoredPeerIDs...),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Explicitly enable/disable server to ensure desired provide mode
|
||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
|
||||
|
||||
bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
|
||||
@ -335,17 +335,18 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
|
||||
recordLifetime = d
|
||||
}
|
||||
|
||||
isBitswapEnabled := cfg.Bitswap.Enabled.WithDefault(config.DefaultBitswapEnabled)
|
||||
isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
|
||||
isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
|
||||
// Don't provide from bitswap when the strategic provider service is active
|
||||
shouldBitswapProvide := isBitswapEnabled && isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
|
||||
|
||||
// Don't provide from bitswap when the legacy noop experiment "strategic provider service" is active
|
||||
isBitswapServerEnabled = isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
|
||||
|
||||
return fx.Options(
|
||||
fx.Provide(BitswapOptions(cfg)),
|
||||
fx.Provide(Bitswap(shouldBitswapProvide)),
|
||||
fx.Provide(OnlineExchange(isBitswapEnabled)),
|
||||
fx.Provide(Bitswap(isBitswapServerEnabled)),
|
||||
fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
|
||||
// Replace our Exchange with a Providing exchange!
|
||||
fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
|
||||
fx.Decorate(ProvidingExchange(isBitswapServerEnabled)),
|
||||
fx.Provide(DNSResolver),
|
||||
fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
|
||||
fx.Provide(Peering),
|
||||
|
||||
@ -2,6 +2,7 @@ package libp2p
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-datastore"
|
||||
@ -29,21 +30,44 @@ type RoutingOptionArgs struct {
|
||||
|
||||
type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error)
|
||||
|
||||
var noopRouter = routinghelpers.Null{}
|
||||
|
||||
func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) {
|
||||
var routers []*routinghelpers.ParallelRouter
|
||||
httpRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
|
||||
|
||||
// Use config.DefaultHTTPRouters if custom override was sent via config.EnvHTTPRouters
|
||||
// or if user did not set any preference in cfg.Routing.DelegatedRouters
|
||||
var httpRouterEndpoints []string
|
||||
if os.Getenv(config.EnvHTTPRouters) != "" || len(cfg.Routing.DelegatedRouters) == 0 {
|
||||
httpRouterEndpoints = config.DefaultHTTPRouters
|
||||
} else {
|
||||
httpRouterEndpoints = cfg.Routing.DelegatedRouters
|
||||
}
|
||||
|
||||
// Append HTTP routers for additional speed
|
||||
for _, endpoint := range config.DefaultHTTPRouters {
|
||||
httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
|
||||
for _, endpoint := range httpRouterEndpoints {
|
||||
httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey, httpRetrievalEnabled)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Mapping router to /routing/v1/* endpoints
|
||||
// https://specs.ipfs.tech/routing/http-routing-v1/
|
||||
r := &irouting.Composer{
|
||||
GetValueRouter: routinghelpers.Null{},
|
||||
PutValueRouter: routinghelpers.Null{},
|
||||
ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide
|
||||
FindPeersRouter: routinghelpers.Null{},
|
||||
FindProvidersRouter: httpRouter,
|
||||
GetValueRouter: httpRouter, // GET /routing/v1/ipns
|
||||
PutValueRouter: httpRouter, // PUT /routing/v1/ipns
|
||||
ProvideRouter: noopRouter, // we don't have spec for sending provides to /routing/v1 (revisit once https://github.com/ipfs/specs/pull/378 or similar is ratified)
|
||||
FindPeersRouter: httpRouter, // /routing/v1/peers
|
||||
FindProvidersRouter: httpRouter, // /routing/v1/providers
|
||||
}
|
||||
|
||||
if endpoint == config.CidContactRoutingURL {
|
||||
// Special-case: cid.contact only supports /routing/v1/providers/cid
|
||||
// we disable other endpoints to avoid sending requests that always fail
|
||||
r.GetValueRouter = noopRouter
|
||||
r.PutValueRouter = noopRouter
|
||||
r.ProvideRouter = noopRouter
|
||||
r.FindPeersRouter = noopRouter
|
||||
}
|
||||
|
||||
routers = append(routers, &routinghelpers.ParallelRouter{
|
||||
@ -119,7 +143,7 @@ func constructDHTRouting(mode dht.ModeOpt) RoutingOption {
|
||||
}
|
||||
|
||||
// ConstructDelegatedRouting is used when Routing.Type = "custom"
|
||||
func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string) RoutingOption {
|
||||
func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string, httpRetrieval bool) RoutingOption {
|
||||
return func(args RoutingOptionArgs) (routing.Routing, error) {
|
||||
return irouting.Parse(routers, methods,
|
||||
&irouting.ExtraDHTParams{
|
||||
@ -130,9 +154,10 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
|
||||
Context: args.Ctx,
|
||||
},
|
||||
&irouting.ExtraHTTPParams{
|
||||
PeerID: peerID,
|
||||
Addrs: httpAddrsFromConfig(addrs),
|
||||
PrivKeyB64: privKey,
|
||||
PeerID: peerID,
|
||||
Addrs: httpAddrsFromConfig(addrs),
|
||||
PrivKeyB64: privKey,
|
||||
HTTPRetrieval: httpRetrieval,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@ -10,25 +10,50 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
|
||||
|
||||
- [Overview](#overview)
|
||||
- [🔦 Highlights](#-highlights)
|
||||
- [Opt-in HTTP Retrieval client](#opt-in-http-retrieval-client)
|
||||
- [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
|
||||
- [Additional new configuration options](#additional-new-configuration-options)
|
||||
- [Grid view in WebUI](#grid-view-in-webui)
|
||||
- [Enhanced DAG-Shaping Controls for `ipfs add`](#enhanced-dag-shaping-controls-for-ipfs-add)
|
||||
- [New `ipfs add` Options](#new-ipfs-add-options)
|
||||
- [Persistent `Import.*` Configuration](#persistent-import-configuration)
|
||||
- [Updated `Import` Profiles](#updated-import-profiles)
|
||||
- [Enhanced DAG-Shaping Controls](#enhanced-dag-shaping-controls)
|
||||
- [New DAG-Shaping `ipfs add` Options](#new-dag-shaping-ipfs-add-options)
|
||||
- [Persistent DAG-Shaping `Import.*` Configuration](#persistent-dag-shaping-import-configuration)
|
||||
- [Updated DAG-Shaping `Import` Profiles](#updated-dag-shaping-import-profiles)
|
||||
- [`Datastore` Metrics Now Opt-In](#datastore-metrics-now-opt-in)
|
||||
- [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
|
||||
- [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
|
||||
- [Pebble Database Format Config](#pebble-database-format-config)
|
||||
- [Improved performance of data onboarding](#improved-performance-of-data-onboarding)
|
||||
- [Fast `ipfs add` in online mode](#fast-ipfs-add-in-online-mode)
|
||||
- [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
|
||||
- [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
|
||||
- [New `Bitswap` configuration options](#new-bitswap-configuration-options)
|
||||
- [New `Routing` configuration options](#new-routing-configuration-options)
|
||||
- [New Pebble database format config](#new-pebble-database-format-config)
|
||||
- [📦️ Important dependency updates](#-important-dependency-updates)
|
||||
- [📝 Changelog](#-changelog)
|
||||
- [👨👩👧👦 Contributors](#-contributors)
|
||||
|
||||
### Overview
|
||||
|
||||
This release brings significant UX and performance improvements to data onboarding, provisioning, and retrieval systems.
|
||||
|
||||
New configuration options let you customize the shape of UnixFS DAGs generated during the data import, control the scope of DAGs announced on the Amino DHT, select which delegated routing endpoints are queried, and choose whether to enable HTTP retrieval alongside Bitswap over Libp2p.
|
||||
|
||||
Continue reading for more details.
|
||||
|
||||
|
||||
### 🔦 Highlights
|
||||
|
||||
#### Opt-in HTTP Retrieval client
|
||||
|
||||
This release adds experimental support for retrieving blocks directly over HTTPS (HTTP/2), complementing the existing Bitswap over Libp2p.
|
||||
|
||||
The opt-in client enables Kubo to use [delegated routing](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) results with `/tls/http` multiaddrs, connecting to HTTPS servers that support [Trustless HTTP Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway)'s Block Responses (`?format=raw`, `application/vnd.ipld.raw`). Fetching blocks via HTTPS (HTTP/2) simplifies infrastructure and reduces costs for storage providers by leveraging HTTP caching and CDNs.
|
||||
|
||||
To enable this feature for testing and feedback, set:
|
||||
|
||||
```console
|
||||
$ ipfs config --json HTTPRetrieval.Enabled true
|
||||
```
|
||||
|
||||
See [`HTTPRetrieval`](https://github.com/ipfs/kubo/blob/master/docs/config.md#httpretrieval) for more details.
|
||||
|
||||
#### Dedicated `Reprovider.Strategy` for MFS
|
||||
|
||||
The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme).
|
||||
@ -39,19 +64,13 @@ Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS al
|
||||
|
||||
See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details.
|
||||
|
||||
#### Additional new configuration options
|
||||
|
||||
- [`Bitswap`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswap) section with `Enabled` and `ServerEnabled` flags determine whether Kubo initializes Bitswap, enabling just the client or both the client and server.
|
||||
- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones.
|
||||
- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content.
|
||||
|
||||
#### Grid view in WebUI
|
||||
|
||||
The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen:
|
||||
|
||||
> 
|
||||
|
||||
#### Enhanced DAG-Shaping Controls for `ipfs add`
|
||||
#### Enhanced DAG-Shaping Controls
|
||||
|
||||
This release advances CIDv1 support by introducing fine-grained control over UnixFS DAG shaping during data ingestion with the `ipfs add` command.
|
||||
|
||||
@ -59,7 +78,7 @@ Wider DAG trees (more links per node, higher fanout, larger thresholds) are bene
|
||||
|
||||
Kubo now allows users to act on these tradeoffs and customize the width of the DAG created by `ipfs add` command.
|
||||
|
||||
##### New `ipfs add` Options
|
||||
##### New DAG-Shaping `ipfs add` Options
|
||||
|
||||
Three new options allow you to override default settings for specific import operations:
|
||||
|
||||
@ -68,7 +87,7 @@ Three new options allow you to override default settings for specific import ope
|
||||
- Note: Directories exceeding this limit or the `Import.UnixFSHAMTDirectorySizeThreshold` are converted to HAMT-based (sharded across multiple blocks) structures.
|
||||
- `--max-hamt-fanout`: Specifies the maximum number of child nodes for HAMT internal structures.
|
||||
|
||||
##### Persistent `Import.*` Configuration
|
||||
##### Persistent DAG-Shaping `Import.*` Configuration
|
||||
|
||||
You can set default values for these options using the following configuration settings:
|
||||
- [`Import.UnixFSFileMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsfilemaxlinks)
|
||||
@ -76,7 +95,7 @@ You can set default values for these options using the following configuration s
|
||||
- [`Import.UnixFSHAMTDirectoryMaxFanout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorymaxfanout)
|
||||
- [`Import.UnixFSHAMTDirectorySizeThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorysizethreshold)
|
||||
|
||||
##### Updated `Import` Profiles
|
||||
##### Updated DAG-Shaping `Import` Profiles
|
||||
|
||||
The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) to incorporate these new `Import.*` settings:
|
||||
- Updated Profile: `test-cid-v1` now includes current defaults as explicit `Import.UnixFSFileMaxLinks=174`, `Import.UnixFSDirectoryMaxLinks=0`, `Import.UnixFSHAMTDirectoryMaxFanout=256` and `Import.UnixFSHAMTDirectorySizeThreshold=256KiB`
|
||||
@ -95,11 +114,55 @@ Convenience opt-in [profiles](https://github.com/ipfs/kubo/blob/master/docs/conf
|
||||
|
||||
It is also possible to manually add the `measure` wrapper. See examples in [`Datastore.Spec`](https://github.com/ipfs/kubo/blob/master/docs/config.md#datastorespec) documentation.
|
||||
|
||||
#### Optimized, dedicated queue for providing fresh CIDs
|
||||
#### Improved performance of data onboarding
|
||||
|
||||
This Kubo release significantly improves both the speed of ingesting data via `ipfs add` and announcing newly produced CIDs to Amino DHT.
|
||||
|
||||
##### Fast `ipfs add` in online mode
|
||||
|
||||
Adding a large directory of data when `ipfs daemon` was running in online mode took a long time. A significant amount of this time was spent writing to and reading from the persisted provider queue. Due to this, many users had to shut down the daemon and perform data import in offline mode. This release fixes this known limitation, significantly improving the speed of `ipfs add`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Performing `ipfs add` of 10GiB file would take about 30 minutes.
|
||||
> Now it takes close to 30 seconds.
|
||||
|
||||
Kubo v0.34:
|
||||
|
||||
```console
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null
|
||||
100.00 MiB / 100.00 MiB [=====================================================================] 100.00%
|
||||
real 0m6.464s
|
||||
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null
|
||||
1000.00 MiB / 1000.00 MiB [===================================================================] 100.00%
|
||||
real 1m10.542s
|
||||
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null
|
||||
10.00 GiB / 10.00 GiB [=======================================================================] 100.00%
|
||||
real 24m5.744s
|
||||
```
|
||||
|
||||
Kubo v0.35:
|
||||
|
||||
```console
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null
|
||||
100.00 MiB / 100.00 MiB [=====================================================================] 100.00%
|
||||
real 0m0.326s
|
||||
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null
|
||||
1.00 GiB / 1.00 GiB [=========================================================================] 100.00%
|
||||
real 0m2.819s
|
||||
|
||||
$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null
|
||||
10.00 GiB / 10.00 GiB [=======================================================================] 100.00%
|
||||
real 0m28.405s
|
||||
```
|
||||
|
||||
##### Optimized, dedicated queue for providing fresh CIDs
|
||||
|
||||
From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0),
|
||||
Bitswap stopped advertising newly added and received blocks to the DHT. Since
|
||||
then `boxo/provider` is responsible for the provide and reprovide logic. Prior
|
||||
then `boxo/provider` is responsible for the first time provide and the recurring reprovide logic. Prior
|
||||
to `v0.35.0`, provides and reprovides were handled together in batches, leading
|
||||
to delays in initial advertisements (provides).
|
||||
|
||||
@ -113,7 +176,7 @@ concurrent provide operations:
|
||||
> [!TIP]
|
||||
> Users who need to provide large volumes of content immediately should consider removing the cap on concurrent provide operations and also set `Routing.AcceleratedDHTClient` to `true`.
|
||||
|
||||
##### Deprecated `ipfs stats provider`
|
||||
###### Deprecated `ipfs stats provider`
|
||||
|
||||
Since the `ipfs stats provider` command was displaying statistics for both
|
||||
provides and reprovides, this command isn't relevant anymore after separating
|
||||
@ -125,7 +188,28 @@ but for reprovides only.
|
||||
> [!NOTE]
|
||||
> `ipfs stats provider` still works, but is marked as deprecated and will be removed in a future release. Be mindful that the command provides only statistics about reprovides (similar to `ipfs stats reprovide`) and not the new provide queue (this will be fixed as a part of wider refactor planned for a future release).
|
||||
|
||||
#### Pebble Database Format Config
|
||||
#### New `Bitswap` configuration options
|
||||
|
||||
- [`Bitswap.Libp2pEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswaplibp2penabled) determines whether Kubo will use Bitswap over libp2p (both client and server).
|
||||
- [`Bitswap.ServerEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswapserverenabled) controls whether Kubo functions as a Bitswap server to host and respond to block requests.
|
||||
- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones.
|
||||
|
||||
#### New `Routing` configuration options
|
||||
|
||||
- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content.
|
||||
- Simplifies testing `HTTPRetrieval.Enabled` in setups where Bitswap over Libp2p and HTTP retrieval is served under different PeerIDs.
|
||||
- [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) allows customizing HTTP routers used by Kubo when `Routing.Type` is set to `auto` or `autoclient`.
|
||||
- Users are now able to adjust the default routing system and directly query custom routers for increased resiliency or when dataset is too big and CIDs are not announced on Amino DHT.
|
||||
|
||||
> [!TIP]
|
||||
>
|
||||
> For example, to use Pinata's routing endpoint in addition to IPNI at `cid.contact`:
|
||||
>
|
||||
> ```console
|
||||
> $ ipfs config --json Routing.DelegatedRouters '["https://cid.contact","https://indexer.pinata.cloud"]'
|
||||
> ```
|
||||
|
||||
#### New Pebble database format config
|
||||
|
||||
This Kubo release provides node operators with more control over [Pebble's `FormatMajorVersion`](https://github.com/cockroachdb/pebble/tree/master?tab=readme-ov-file#format-major-versions). This allows testing a new Kubo release without automatically migrating Pebble datastores, keeping the ability to switch back to older Kubo.
|
||||
|
||||
|
||||
150
docs/config.md
150
docs/config.md
@ -37,7 +37,7 @@ config file at runtime.
|
||||
- [`AutoTLS.RegistrationDelay`](#autotlsregistrationdelay)
|
||||
- [`AutoTLS.CAEndpoint`](#autotlscaendpoint)
|
||||
- [`Bitswap`](#bitswap)
|
||||
- [`Bitswap.Enabled`](#bitswapenabled)
|
||||
- [`Bitswap.Libp2pEnabled`](#bitswaplibp2penabled)
|
||||
- [`Bitswap.ServerEnabled`](#bitswapserverenabled)
|
||||
- [`Bootstrap`](#bootstrap)
|
||||
- [`Datastore`](#datastore)
|
||||
@ -127,6 +127,7 @@ config file at runtime.
|
||||
- [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient)
|
||||
- [`Routing.LoopbackAddressesOnLanDHT`](#routingloopbackaddressesonlandht)
|
||||
- [`Routing.IgnoreProviders`](#routingignoreproviders)
|
||||
- [`Routing.DelegatedRouters`](#routingdelegatedrouters)
|
||||
- [`Routing.Routers`](#routingrouters)
|
||||
- [`Routing.Routers: Type`](#routingrouters-type)
|
||||
- [`Routing.Routers: Parameters`](#routingrouters-parameters)
|
||||
@ -184,6 +185,13 @@ config file at runtime.
|
||||
- [`DNS`](#dns)
|
||||
- [`DNS.Resolvers`](#dnsresolvers)
|
||||
- [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
|
||||
- [`HTTPRetrieval`](#httpretrieval)
|
||||
- [`HTTPRetrieval.Enabled`](#httpretrievalenabled)
|
||||
- [`HTTPRetrieval.Allowlist`](#httpretrievalallowlist)
|
||||
- [`HTTPRetrieval.Denylist`](#httpretrievaldenylist)
|
||||
- [`HTTPRetrieval.NumWorkers`](#httpretrievalnumworkers)
|
||||
- [`HTTPRetrieval.MaxBlockSize`](#httpretrievalmaxblocksize)
|
||||
- [`HTTPRetrieval.TLSInsecureSkipVerify`](#httpretrievaltlsinsecureskipverify)
|
||||
- [`Import`](#import)
|
||||
- [`Import.CidVersion`](#importcidversion)
|
||||
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
|
||||
@ -624,16 +632,20 @@ Type: `optionalString`
|
||||
|
||||
## `Bitswap`
|
||||
|
||||
High level client and server configuration of the [Bitswap Protocol](https://specs.ipfs.tech/bitswap-protocol/).
|
||||
High level client and server configuration of the [Bitswap Protocol](https://specs.ipfs.tech/bitswap-protocol/) over libp2p.
|
||||
|
||||
For internal configuration see [`Internal.Bitswap`](#internalbitswap).
|
||||
|
||||
### `Bitswap.Enabled`
|
||||
For HTTP version see [`HTTPRetrieval`](#httpretrieval).
|
||||
|
||||
Manages both Bitswap client and server functionality. For testing or operating a node without Bitswap requirements.
|
||||
### `Bitswap.Libp2pEnabled`
|
||||
|
||||
Determines whether Kubo will use Bitswap over libp2p.
|
||||
|
||||
Disabling this, will remove `/ipfs/bitswap/*` protocol support from [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses, effectively shutting down both Bitswap libp2p client and server.
|
||||
|
||||
> [!WARNING]
|
||||
> Bitswap is a core component of Kubo, and disabling it completely may cause unpredictable outcomes. Treat this as experimental and use it solely for testing purposes.
|
||||
> Bitswap over libp2p is a core component of Kubo and the oldest way of exchanging blocks. Disabling it completely may cause unpredictable outcomes, such as retrieval failures, if the only providers were libp2p ones. Treat this as experimental and use it solely for testing purposes with `HTTPRetrieval.Enabled`.
|
||||
|
||||
Default: `true`
|
||||
|
||||
@ -643,9 +655,9 @@ Type: `flag`
|
||||
|
||||
Determines whether Kubo functions as a Bitswap server to host and respond to block requests.
|
||||
|
||||
Disabling the server retains client and protocol support in libp2p identify responses but causes Kubo to reply with "don't have" to all block requests.
|
||||
Disabling the server retains client and protocol support in [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses but causes Kubo to reply with "don't have" to all block requests.
|
||||
|
||||
Default: `true`
|
||||
Default: `true` (requires `Bitswap.Libp2pEnabled`)
|
||||
|
||||
Type: `flag`
|
||||
|
||||
@ -1725,7 +1737,7 @@ Contains options for content, peer, and IPNS routing mechanisms.
|
||||
There are multiple routing options: "auto", "autoclient", "none", "dht", "dhtclient", and "custom".
|
||||
|
||||
* **DEFAULT:** If unset, or set to "auto", your node will use the public IPFS DHT (aka "Amino")
|
||||
and parallel HTTP routers listed below for additional speed.
|
||||
and parallel [`Routing.DelegatedRouters`](#routingdelegatedrouters) for additional speed.
|
||||
|
||||
* If set to "autoclient", your node will behave as in "auto" but without running a DHT server.
|
||||
|
||||
@ -1755,15 +1767,13 @@ To force a specific Amino DHT-only mode, client or server, set `Routing.Type` to
|
||||
unless you're sure your node is reachable from the public network.
|
||||
|
||||
When `Routing.Type` is set to `auto` or `autoclient` your node will accelerate some types of routing
|
||||
by leveraging HTTP endpoints compatible with [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/)
|
||||
by leveraging [`Routing.DelegatedRouters`](#routingdelegatedrouters) HTTP endpoints compatible with [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/)
|
||||
introduced in [IPIP-337](https://github.com/ipfs/specs/pull/337)
|
||||
in addition to the Amino DHT.
|
||||
By default, an instance of [IPNI](https://github.com/ipni/specs/blob/main/IPNI.md#readme)
|
||||
at https://cid.contact is used.
|
||||
|
||||
Alternative routing rules can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`.
|
||||
[Advanced routing rules](https://github.com/ipfs/kubo/blob/master/docs/delegated-routing.md) can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`.
|
||||
|
||||
Default: `auto` (DHT + IPNI)
|
||||
Default: `auto` (DHT + [`Routing.DelegatedRouters`](#routingdelegatedrouters))
|
||||
|
||||
Type: `optionalString` (`null`/missing means the default)
|
||||
|
||||
@ -1828,16 +1838,32 @@ Type: `bool` (missing means `false`)
|
||||
|
||||
### `Routing.IgnoreProviders`
|
||||
|
||||
An array of peerIDs. Any provider record associated to one of these peer IDs is ignored.
|
||||
An array of [string-encoded PeerIDs](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation). Any provider record associated to one of these peer IDs is ignored.
|
||||
|
||||
Apart from ignoring specific providers for reasons like misbehaviour etc. this
|
||||
setting is useful to ignore providers as a way to indicate preference, when the same provider
|
||||
is found under different peerIDs (i.e. one for HTTP and one for Bitswap retrieval).
|
||||
|
||||
> [!TIP]
|
||||
> This denylist operates on PeerIDs.
|
||||
> To deny specific HTTP Provider URL, use [`HTTPRetrieval.Denylist`](#httpretrievaldenylist) instead.
|
||||
|
||||
Default: `[]`
|
||||
|
||||
Type: `array[peerID]`
|
||||
Type: `array[string]`
|
||||
|
||||
### `Routing.DelegatedRouters`
|
||||
|
||||
This is an array of URL hostnames that support the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) which are used alongside the DHT when [`Routing.Type`](#routingtype) is set to `auto` or `autoclient`.
|
||||
|
||||
> [!TIP]
|
||||
> Delegated routing allows IPFS implementations to offload tasks like content routing, peer routing, and naming to a separate process or server while also benefiting from HTTP caching.
|
||||
>
|
||||
> One can run their own delegated router either by implementing the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) themselves, or by using [Someguy](https://github.com/ipfs/someguy), a turn-key implementation that proxies requests to the Amino DHT and other delegated routing servers, such as the Network Indexer at `cid.contact`. Public utility instance of Someguy is hosted at [`https://delegated-ipfs.dev`](https://docs.ipfs.tech/concepts/public-utilities/#delegated-routing).
|
||||
|
||||
Default: `["https://cid.contact"]` (empty or `nil` will also use this default; to disable delegated routing, set `Routing.Type` to `dht` or `dhtclient`)
|
||||
|
||||
Type: `array[string]`
|
||||
|
||||
### `Routing.Routers`
|
||||
|
||||
@ -2596,6 +2622,100 @@ Default: Respect DNS Response TTL
|
||||
|
||||
Type: `optionalDuration`
|
||||
|
||||
## `HTTPRetrieval`
|
||||
|
||||
`HTTPRetrieval` is configuration for pure HTTP retrieval based on Trustless HTTP Gateways'
|
||||
[Block Responses (`application/vnd.ipld.raw`)](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw)
|
||||
which can be used in addition to or instead of retrieving blocks with [Bitswap over Libp2p](#bitswap).
|
||||
|
||||
Default: `{}`
|
||||
|
||||
Type: `object`
|
||||
|
||||
### `HTTPRetrieval.Enabled`
|
||||
|
||||
> [!CAUTION]
|
||||
> This feature is **EXPERIMENTAL** and may change in future release. Enable with caution, and provide feedback via GitHub issues.
|
||||
|
||||
Controls whether HTTP-based block retrieval is enabled.
|
||||
|
||||
When enabled, Kubo will be able to act on `/tls/http` (HTTP/2) providers ([Trustless HTTP Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/)) returned by the [`Routing.DelegatedRouters`](#routingdelegatedrouters)
|
||||
to perform pure HTTP [block retrievals](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw)
|
||||
in addition to [Bitswap over Libp2p](#bitswap).
|
||||
|
||||
HTTP requests for `application/vnd.ipld.raw` will be issued instead of Bitswap if a peer has a `/tls/http` multiaddr
|
||||
and the HTTPS server returns HTTP 200 for the [probe path](https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> - Requires TLS and HTTP/2.
|
||||
> - This feature works in the same way as Bitswap: connected HTTP-peers receive optimistic block requests even for content that they are not announcing.
|
||||
> - HTTP client does not follow redirects. Providers should keep announcements up to date.
|
||||
> - IPFS ecosystem is working towards [supporting HTTP providers on Amino DHT](https://github.com/ipfs/specs/issues/496). Currently, HTTP providers are mostly limited to results from [`Routing.DelegatedRouters`](#routingdelegatedrouters) endpoints and requires `Routing.Type=auto|autoclient`.
|
||||
|
||||
Default: `false`
|
||||
|
||||
Type: `flag`
|
||||
|
||||
### `HTTPRetrieval.Allowlist`
|
||||
|
||||
Optional list of hostnames for which HTTP retrieval is allowed for.
|
||||
If this list is not empty, only hosts matching these entries will be allowed for HTTP retrieval.
|
||||
|
||||
> [!TIP]
|
||||
> To limit HTTP retrieval to a provider at `/dns4/example.com/tcp/443/tls/http` (which would serve `HEAD|GET https://example.com/ipfs/cid?format=raw`), set this to `["example.com"]`
|
||||
|
||||
Default: `[]`
|
||||
|
||||
Type: `array[string]`
|
||||
|
||||
### `HTTPRetrieval.Denylist`
|
||||
|
||||
Optional list of hostnames for which HTTP retrieval is not allowed.
|
||||
Denylist entries take precedence over Allowlist entries.
|
||||
|
||||
|
||||
> [!TIP]
|
||||
> This denylist operates on HTTP endpoint hostnames.
|
||||
> To deny specific PeerID, use [`Routing.IgnoreProviders`](#routingignoreproviders) instead.
|
||||
|
||||
Default: `[]`
|
||||
|
||||
Type: `array[string]`
|
||||
|
||||
### `HTTPRetrieval.NumWorkers`
|
||||
|
||||
The number of worker goroutines to use for concurrent HTTP retrieval operations.
|
||||
This setting controls the level of parallelism for HTTP-based block retrieval operations.
|
||||
Higher values can improve performance when retrieving many blocks but may increase resource usage.
|
||||
|
||||
Default: `16`
|
||||
|
||||
Type: `optionalInteger`
|
||||
|
||||
### `HTTPRetrieval.MaxBlockSize`
|
||||
|
||||
Sets the maximum size of a block that the HTTP retrieval client will accept.
|
||||
|
||||
> [!NOTE]
|
||||
> This setting is a security feature designed to protect Kubo from malicious providers who might send excessively large or invalid data.
|
||||
> Increasing this value allows Kubo to retrieve larger blocks from compatible HTTP providers, but doing so reduces interoperability with Bitswap, and increases potential security risks.
|
||||
>
|
||||
> Learn more: [Supporting Large IPLD Blocks: Why block limits?](https://discuss.ipfs.tech/t/supporting-large-ipld-blocks/15093#why-block-limits-5)
|
||||
|
||||
Default: `2MiB` (matching [Bitswap size limit](https://specs.ipfs.tech/bitswap-protocol/#block-sizes))
|
||||
|
||||
Type: `optionalString`
|
||||
|
||||
### `HTTPRetrieval.TLSInsecureSkipVerify`
|
||||
|
||||
Disables TLS certificate validation.
|
||||
Allows making HTTPS connections to HTTP/2 test servers with self-signed TLS certificates.
|
||||
Only for testing, do not use in production.
|
||||
|
||||
Default: `false`
|
||||
|
||||
Type: `flag`
|
||||
|
||||
## `Import`
|
||||
|
||||
Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.
|
||||
|
||||
@ -16,6 +16,14 @@ which then got changed and standardized as [Routing V1 HTTP API](https://specs.i
|
||||
|
||||
Kubo 0.23.0 release added support for [self-hosting Routing V1 HTTP API server](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.23.md#self-hosting-routingv1-endpoint-for-delegated-routing-needs).
|
||||
|
||||
|
||||
> [!TIP]
|
||||
> Kubo 0.35 added support for [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
|
||||
>
|
||||
> Most of users are best served by setting delegated HTTP router URLs there and `Routing.Type` to `auto` or `autoclient`, rather than custom routing with complex `Routing.Routers` and `Routing.Methods` directly.
|
||||
>
|
||||
> The rest of this documentation should be considered only by advanced users and researchers.
|
||||
|
||||
Now we need a better way to add different routers using different protocols
|
||||
like [Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) or Amino
|
||||
DHT, and be able to configure them (future routing systems to come) to cover different use cases.
|
||||
|
||||
@ -149,12 +149,13 @@ func parse(visited map[string]bool,
|
||||
}
|
||||
|
||||
type ExtraHTTPParams struct {
|
||||
PeerID string
|
||||
Addrs []string
|
||||
PrivKeyB64 string
|
||||
PeerID string
|
||||
Addrs []string
|
||||
PrivKeyB64 string
|
||||
HTTPRetrieval bool
|
||||
}
|
||||
|
||||
func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string) (routing.Routing, error) {
|
||||
func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string, httpRetrieval bool) (routing.Routing, error) {
|
||||
return httpRoutingFromConfig(
|
||||
config.Router{
|
||||
Type: "http",
|
||||
@ -163,9 +164,10 @@ func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey
|
||||
},
|
||||
},
|
||||
&ExtraHTTPParams{
|
||||
PeerID: peerID,
|
||||
Addrs: addrs,
|
||||
PrivKeyB64: privKey,
|
||||
PeerID: peerID,
|
||||
Addrs: addrs,
|
||||
PrivKeyB64: privKey,
|
||||
HTTPRetrieval: httpRetrieval,
|
||||
},
|
||||
)
|
||||
}
|
||||
@ -200,13 +202,18 @@ func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (rout
|
||||
return nil, err
|
||||
}
|
||||
|
||||
protocols := config.DefaultHTTPRoutersFilterProtocols
|
||||
if extraHTTP.HTTPRetrieval {
|
||||
protocols = append(protocols, "transport-ipfs-gateway-http")
|
||||
}
|
||||
|
||||
cli, err := drclient.New(
|
||||
params.Endpoint,
|
||||
drclient.WithHTTPClient(delegateHTTPClient),
|
||||
drclient.WithIdentity(key),
|
||||
drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs),
|
||||
drclient.WithUserAgent(version.GetUserAgentVersion()),
|
||||
drclient.WithProtocolFilter(config.DefaultHTTPRoutersFilterProtocols),
|
||||
drclient.WithProtocolFilter(protocols),
|
||||
drclient.WithStreamResultsRequired(), // https://specs.ipfs.tech/routing/http-routing-v1/#streaming
|
||||
drclient.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484
|
||||
)
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/boxo/bitswap/network/bsnet"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
@ -61,32 +63,32 @@ func TestBitswapConfig(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("server disabled and client enabled", func(t *testing.T) {
|
||||
t.Run("client still works when server disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := harness.NewT(t)
|
||||
|
||||
provider := h.NewNode().Init()
|
||||
provider.SetIPFSConfig("Bitswap.ServerEnabled", false)
|
||||
provider.StartDaemon()
|
||||
requester := h.NewNode().Init()
|
||||
requester.SetIPFSConfig("Bitswap.ServerEnabled", false)
|
||||
requester.StartDaemon()
|
||||
|
||||
requester := h.NewNode().Init().StartDaemon()
|
||||
hash := requester.IPFSAddStr(string(testData))
|
||||
|
||||
provider.Connect(requester)
|
||||
provider := h.NewNode().Init().StartDaemon()
|
||||
hash := provider.IPFSAddStr(string(testData))
|
||||
requester.Connect(provider)
|
||||
|
||||
// Even when the server is disabled, the client should be able to retrieve data
|
||||
res := provider.RunIPFS("cat", hash)
|
||||
res := requester.RunIPFS("cat", hash)
|
||||
assert.Equal(t, testData, res.Stdout.Bytes(), "retrieved data should match original")
|
||||
})
|
||||
|
||||
t.Run("bitswap completely disabled", func(t *testing.T) {
|
||||
t.Run("bitswap over libp2p disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := harness.NewT(t)
|
||||
|
||||
requester := h.NewNode().Init()
|
||||
requester.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Bitswap.Enabled = config.False
|
||||
cfg.Bitswap.Libp2pEnabled = config.False
|
||||
cfg.Bitswap.ServerEnabled = config.False
|
||||
cfg.HTTPRetrieval.Enabled = config.True
|
||||
})
|
||||
requester.StartDaemon()
|
||||
|
||||
@ -112,27 +114,63 @@ func TestBitswapConfig(t *testing.T) {
|
||||
assert.Equal(t, []byte("random"), res.Stdout.Bytes(), "cat should return the added data")
|
||||
})
|
||||
|
||||
// TODO: Disabling Bitswap.Enabled should remove /ifps/bitswap* protocols from `ipfs id`
|
||||
// t.Run("bitswap protocols disabled", func(t *testing.T) {
|
||||
// t.Parallel()
|
||||
// harness.EnableDebugLogging()
|
||||
// h := harness.NewT(t)
|
||||
// Disabling Bitswap.Libp2pEnabled should remove /ipfs/bitswap* protocols from `ipfs id`
|
||||
t.Run("disabling bitswap over libp2p removes it from identify protocol list", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := harness.NewT(t)
|
||||
|
||||
// provider := h.NewNode().Init()
|
||||
// provider.SetIPFSConfig("Bitswap.ServerEnabled", false)
|
||||
// provider = provider.StartDaemon()
|
||||
// requester := h.NewNode().Init().StartDaemon()
|
||||
// requester.Connect(provider)
|
||||
// // Parse and check ID output
|
||||
// res := provider.IPFS("id", "-f", "<protocols>")
|
||||
// protocols := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n")
|
||||
provider := h.NewNode().Init()
|
||||
provider.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.Bitswap.Libp2pEnabled = config.False
|
||||
cfg.Bitswap.ServerEnabled = config.False
|
||||
cfg.HTTPRetrieval.Enabled = config.True
|
||||
})
|
||||
provider = provider.StartDaemon()
|
||||
requester := h.NewNode().Init().StartDaemon()
|
||||
requester.Connect(provider)
|
||||
|
||||
// // No bitswap protocols should be present
|
||||
// for _, proto := range protocols {
|
||||
// assert.NotContains(t, proto, bsnet.ProtocolBitswap, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
// assert.NotContains(t, proto, bsnet.ProtocolBitswapNoVers, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
// assert.NotContains(t, proto, bsnet.ProtocolBitswapOneOne, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
// assert.NotContains(t, proto, bsnet.ProtocolBitswapOneZero, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
// }
|
||||
// })
|
||||
// read libp2p identify from remote peer, and print protocols
|
||||
res := requester.IPFS("id", "-f", "<protocols>", provider.PeerID().String())
|
||||
protocols := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n")
|
||||
|
||||
// No bitswap protocols should be present
|
||||
for _, proto := range protocols {
|
||||
assert.NotContains(t, proto, bsnet.ProtocolBitswap, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
assert.NotContains(t, proto, bsnet.ProtocolBitswapNoVers, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
assert.NotContains(t, proto, bsnet.ProtocolBitswapOneOne, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
assert.NotContains(t, proto, bsnet.ProtocolBitswapOneZero, "bitswap protocol %s should not be advertised when server is disabled", proto)
|
||||
}
|
||||
})
|
||||
|
||||
// HTTPRetrieval uses bitswap engine, we need it
|
||||
t.Run("errors when both HTTP and libp2p are disabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// init Kubo repo
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.HTTPRetrieval.Enabled = config.False
|
||||
cfg.Bitswap.Libp2pEnabled = config.False
|
||||
cfg.Bitswap.ServerEnabled = config.Default
|
||||
})
|
||||
res := node.RunIPFS("daemon")
|
||||
assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
})
|
||||
|
||||
// HTTPRetrieval uses bitswap engine, we need it
|
||||
t.Run("errors when user set conflicting HTTP and libp2p flags", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// init Kubo repo
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
cfg.HTTPRetrieval.Enabled = config.False
|
||||
cfg.Bitswap.Libp2pEnabled = config.False
|
||||
cfg.Bitswap.ServerEnabled = config.True // bad user config: cant enable server when libp2p is down
|
||||
})
|
||||
res := node.RunIPFS("daemon")
|
||||
assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
})
|
||||
}
|
||||
|
||||
@ -1,69 +1,20 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/boxo/ipns"
|
||||
"github.com/ipfs/boxo/routing/http/server"
|
||||
"github.com/ipfs/boxo/routing/http/types"
|
||||
"github.com/ipfs/boxo/routing/http/types/iter"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/routing"
|
||||
"github.com/ipfs/kubo/test/cli/testutils/httprouting"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type fakeHTTPContentRouter struct {
|
||||
m sync.Mutex
|
||||
provideBitswapCalls int
|
||||
findProvidersCalls int
|
||||
findPeersCalls int
|
||||
}
|
||||
|
||||
func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.findProvidersCalls++
|
||||
return iter.FromSlice([]iter.Result[types.Record]{}), nil
|
||||
}
|
||||
|
||||
// nolint deprecated
|
||||
func (r *fakeHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.provideBitswapCalls++
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (r *fakeHTTPContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.findPeersCalls++
|
||||
return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil
|
||||
}
|
||||
|
||||
func (r *fakeHTTPContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) {
|
||||
return nil, routing.ErrNotSupported
|
||||
}
|
||||
|
||||
func (r *fakeHTTPContentRouter) PutIPNS(ctx context.Context, name ipns.Name, rec *ipns.Record) error {
|
||||
return routing.ErrNotSupported
|
||||
}
|
||||
|
||||
func (r *fakeHTTPContentRouter) numFindProvidersCalls() int {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
return r.findProvidersCalls
|
||||
}
|
||||
|
||||
// userAgentRecorder records the user agent of every HTTP request
|
||||
type userAgentRecorder struct {
|
||||
delegate http.Handler
|
||||
@ -76,16 +27,19 @@ func (r *userAgentRecorder) ServeHTTP(w http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
|
||||
func TestContentRoutingHTTP(t *testing.T) {
|
||||
cr := &fakeHTTPContentRouter{}
|
||||
mockRouter := &httprouting.MockHTTPContentRouter{}
|
||||
|
||||
// run the content routing HTTP server
|
||||
userAgentRecorder := &userAgentRecorder{delegate: server.Handler(cr)}
|
||||
userAgentRecorder := &userAgentRecorder{delegate: server.Handler(mockRouter)}
|
||||
server := httptest.NewServer(userAgentRecorder)
|
||||
t.Cleanup(func() { server.Close() })
|
||||
|
||||
// setup the node
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.Runner.Env["IPFS_HTTP_ROUTERS"] = server.URL
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
// setup Kubo node to use mocked HTTP Router
|
||||
cfg.Routing.DelegatedRouters = []string{server.URL}
|
||||
})
|
||||
node.StartDaemon()
|
||||
|
||||
// compute a random CID
|
||||
@ -107,7 +61,7 @@ func TestContentRoutingHTTP(t *testing.T) {
|
||||
|
||||
// verify the content router was called
|
||||
assert.Eventually(t, func() bool {
|
||||
return cr.numFindProvidersCalls() > 0
|
||||
return mockRouter.NumFindProvidersCalls() > 0
|
||||
}, time.Minute, 10*time.Millisecond)
|
||||
|
||||
assert.NotEmpty(t, userAgentRecorder.userAgents)
|
||||
|
||||
145
test/cli/http_retrieval_client_test.go
Normal file
145
test/cli/http_retrieval_client_test.go
Normal file
@ -0,0 +1,145 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/boxo/routing/http/server"
|
||||
"github.com/ipfs/boxo/routing/http/types"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
"github.com/ipfs/kubo/test/cli/testutils/httprouting"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestHTTPRetrievalClient(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// many moving pieces here, show more when debug is needed
|
||||
debug := os.Getenv("DEBUG") == "true"
|
||||
|
||||
// usee local /routing/v1/providers/{cid} and
|
||||
// /ipfs/{cid} HTTP servers to confirm HTTP-only retrieval works end-to-end.
|
||||
t.Run("works end-to-end with an HTTP-only provider", func(t *testing.T) {
|
||||
// setup mocked HTTP Router to handle /routing/v1/providers/cid
|
||||
mockRouter := &httprouting.MockHTTPContentRouter{Debug: debug}
|
||||
delegatedRoutingServer := httptest.NewServer(server.Handler(mockRouter))
|
||||
t.Cleanup(func() { delegatedRoutingServer.Close() })
|
||||
|
||||
// init Kubo repo
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
|
||||
node.UpdateConfig(func(cfg *config.Config) {
|
||||
// explicitly enable http client
|
||||
cfg.HTTPRetrieval.Enabled = config.True
|
||||
// allow NewMockHTTPProviderServer to use self-signed TLS cert
|
||||
cfg.HTTPRetrieval.TLSInsecureSkipVerify = config.True
|
||||
// setup client-only routing which asks both HTTP + DHT
|
||||
// cfg.Routing.Type = config.NewOptionalString("autoclient")
|
||||
// setup Kubo node to use mocked HTTP Router
|
||||
cfg.Routing.DelegatedRouters = []string{delegatedRoutingServer.URL}
|
||||
})
|
||||
|
||||
// compute a random CID
|
||||
randStr := string(testutils.RandomBytes(100))
|
||||
res := node.PipeStrToIPFS(randStr, "add", "-qn", "--cid-version", "1") // -n means dont add to local repo, just produce CID
|
||||
wantCIDStr := res.Stdout.Trimmed()
|
||||
testCid := cid.MustParse(wantCIDStr)
|
||||
|
||||
// setup mock HTTP provider
|
||||
httpProviderServer := NewMockHTTPProviderServer(testCid, randStr, debug)
|
||||
t.Cleanup(func() { httpProviderServer.Close() })
|
||||
httpHost, httpPort, err := splitHostPort(httpProviderServer.URL)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// setup /routing/v1/providers/cid result that points at our mocked HTTP provider
|
||||
mockHTTPProviderPeerID := "12D3KooWCjfPiojcCUmv78Wd1NJzi4Mraj1moxigp7AfQVQvGLwH" // static, it does not matter, we only care about multiaddr
|
||||
mockHTTPMultiaddr, _ := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s/tls/http", httpHost, httpPort))
|
||||
mpid, _ := peer.Decode(mockHTTPProviderPeerID)
|
||||
mockRouter.AddProvider(testCid, &types.PeerRecord{
|
||||
Schema: types.SchemaPeer,
|
||||
ID: &mpid,
|
||||
Addrs: []types.Multiaddr{{Multiaddr: mockHTTPMultiaddr}},
|
||||
// no explicit Protocols, ensure multiaddr alone is enough
|
||||
})
|
||||
|
||||
// Start Kubo
|
||||
node.StartDaemon()
|
||||
|
||||
if debug {
|
||||
fmt.Printf("delegatedRoutingServer.URL: %s\n", delegatedRoutingServer.URL)
|
||||
fmt.Printf("httpProviderServer.URL: %s\n", httpProviderServer.URL)
|
||||
fmt.Printf("httpProviderServer.Multiaddr: %s\n", mockHTTPMultiaddr)
|
||||
fmt.Printf("testCid: %s\n", testCid)
|
||||
}
|
||||
|
||||
// Now, make Kubo to read testCid. it was not added to local blockstore, so it has only one provider -- a HTTP server.
|
||||
|
||||
// First, confirm delegatedRoutingServer returned HTTP provider
|
||||
findprovsRes := node.IPFS("routing", "findprovs", testCid.String())
|
||||
assert.Equal(t, mockHTTPProviderPeerID, findprovsRes.Stdout.Trimmed())
|
||||
|
||||
// Ok, now attempt retrieval.
|
||||
// If there was no timeout and returned bytes match expected body, HTTP routing and retrieval worked end-to-end.
|
||||
catRes := node.IPFS("cat", testCid.String())
|
||||
assert.Equal(t, randStr, catRes.Stdout.Trimmed())
|
||||
})
|
||||
}
|
||||
|
||||
// NewMockHTTPProviderServer pretends to be http provider that supports
|
||||
// block response https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
|
||||
func NewMockHTTPProviderServer(c cid.Cid, body string, debug bool) *httptest.Server {
|
||||
expectedPathPrefix := "/ipfs/" + c.String()
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if debug {
|
||||
fmt.Printf("NewMockHTTPProviderServer GET %s\n", req.URL.Path)
|
||||
}
|
||||
if strings.HasPrefix(req.URL.Path, expectedPathPrefix) {
|
||||
w.Header().Set("Content-Type", "application/vnd.ipld.raw")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if req.Method == "GET" {
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "NewMockHTTPProviderServer GET %s error: %v\n", req.URL.Path, err)
|
||||
}
|
||||
}
|
||||
} else if strings.HasPrefix(req.URL.Path, "/ipfs/bafkqaaa") {
|
||||
// This is probe from https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths
|
||||
w.Header().Set("Content-Type", "application/vnd.ipld.raw")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
} else {
|
||||
http.Error(w, "Not Found", http.StatusNotFound)
|
||||
}
|
||||
})
|
||||
|
||||
// Make it HTTP/2 with self-signed TLS cert
|
||||
srv := httptest.NewUnstartedServer(handler)
|
||||
srv.EnableHTTP2 = true
|
||||
srv.StartTLS()
|
||||
return srv
|
||||
}
|
||||
|
||||
func splitHostPort(httpUrl string) (ipAddr string, port string, err error) {
|
||||
u, err := url.Parse(httpUrl)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.Scheme == "" || u.Host == "" {
|
||||
return "", "", fmt.Errorf("invalid URL format: missing scheme or host")
|
||||
}
|
||||
ipAddr, port, err = net.SplitHostPort(u.Host)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to split host and port from %q: %w", u.Host, err)
|
||||
}
|
||||
return ipAddr, port, nil
|
||||
}
|
||||
117
test/cli/testutils/httprouting/mock_http_content_router.go
Normal file
117
test/cli/testutils/httprouting/mock_http_content_router.go
Normal file
@ -0,0 +1,117 @@
|
||||
package httprouting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/boxo/ipns"
|
||||
"github.com/ipfs/boxo/routing/http/server"
|
||||
"github.com/ipfs/boxo/routing/http/types"
|
||||
"github.com/ipfs/boxo/routing/http/types/iter"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/routing"
|
||||
)
|
||||
|
||||
// MockHTTPContentRouter provides /routing/v1
|
||||
// (https://specs.ipfs.tech/routing/http-routing-v1/) server implementation
|
||||
// based on github.com/ipfs/boxo/routing/http/server
|
||||
type MockHTTPContentRouter struct {
|
||||
m sync.Mutex
|
||||
provideBitswapCalls int
|
||||
findProvidersCalls int
|
||||
findPeersCalls int
|
||||
providers map[cid.Cid][]types.Record
|
||||
peers map[peer.ID][]*types.PeerRecord
|
||||
Debug bool
|
||||
}
|
||||
|
||||
func (r *MockHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) {
|
||||
if r.Debug {
|
||||
fmt.Printf("MockHTTPContentRouter.FindProviders(%s)\n", key.String())
|
||||
}
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.findProvidersCalls++
|
||||
if r.providers == nil {
|
||||
r.providers = make(map[cid.Cid][]types.Record)
|
||||
}
|
||||
records, found := r.providers[key]
|
||||
if !found {
|
||||
return iter.FromSlice([]iter.Result[types.Record]{}), nil
|
||||
}
|
||||
results := make([]iter.Result[types.Record], len(records))
|
||||
for i, rec := range records {
|
||||
results[i] = iter.Result[types.Record]{Val: rec}
|
||||
if r.Debug {
|
||||
fmt.Printf("MockHTTPContentRouter.FindProviders(%s) result: %+v\n", key.String(), rec)
|
||||
}
|
||||
}
|
||||
return iter.FromSlice(results), nil
|
||||
}
|
||||
|
||||
// nolint deprecated
|
||||
func (r *MockHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.provideBitswapCalls++
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (r *MockHTTPContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
r.findPeersCalls++
|
||||
|
||||
if r.peers == nil {
|
||||
r.peers = make(map[peer.ID][]*types.PeerRecord)
|
||||
}
|
||||
records, found := r.peers[pid]
|
||||
if !found {
|
||||
return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil
|
||||
}
|
||||
|
||||
results := make([]iter.Result[*types.PeerRecord], len(records))
|
||||
for i, rec := range records {
|
||||
results[i] = iter.Result[*types.PeerRecord]{Val: rec}
|
||||
if r.Debug {
|
||||
fmt.Printf("MockHTTPContentRouter.FindPeers(%s) result: %+v\n", pid.String(), rec)
|
||||
}
|
||||
}
|
||||
return iter.FromSlice(results), nil
|
||||
}
|
||||
|
||||
func (r *MockHTTPContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) {
|
||||
return nil, routing.ErrNotSupported
|
||||
}
|
||||
|
||||
func (r *MockHTTPContentRouter) PutIPNS(ctx context.Context, name ipns.Name, rec *ipns.Record) error {
|
||||
return routing.ErrNotSupported
|
||||
}
|
||||
|
||||
func (r *MockHTTPContentRouter) NumFindProvidersCalls() int {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
return r.findProvidersCalls
|
||||
}
|
||||
|
||||
// AddProvider adds a record for a given CID
|
||||
func (r *MockHTTPContentRouter) AddProvider(key cid.Cid, record types.Record) {
|
||||
r.m.Lock()
|
||||
defer r.m.Unlock()
|
||||
if r.providers == nil {
|
||||
r.providers = make(map[cid.Cid][]types.Record)
|
||||
}
|
||||
r.providers[key] = append(r.providers[key], record)
|
||||
|
||||
peerRecord, ok := record.(*types.PeerRecord)
|
||||
if ok {
|
||||
if r.peers == nil {
|
||||
r.peers = make(map[peer.ID][]*types.PeerRecord)
|
||||
}
|
||||
pid := peerRecord.ID
|
||||
r.peers[*pid] = append(r.peers[*pid], peerRecord)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user