mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
feat: built-in content blocking based on IPIP-383 (#10161)
Fixes #8492 This introduces "nopfs" as a preloaded plugin into Kubo with support for denylists from https://github.com/ipfs/specs/pull/383 It automatically makes Kubo watch *.deny files found in: - /etc/ipfs/denylists - $XDG_CONFIG_HOME/ipfs/denylists - $IPFS_PATH/denylists * test: Gateway.NoFetch and GatewayOverLibp2p adds missing tests for "no fetch" gateways one can expose, in both cases the offline mode is done by passing custom blockservice/exchange into path resolver, which means global path resolver that has nopfs intercept is not used, and the content blocking does not happen on these gateways. * fix: use offline path resolvers where appropriate this fixes the problem described in https://github.com/ipfs/kubo/pull/10161#issuecomment-1782175955 by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones --------- Co-authored-by: Henrique Dias <hacdias@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
This commit is contained in:
Hector Sanjuan
GitHub
parent
4f303d3208
commit
a0f34b16dd
@ -30,6 +30,7 @@ Featureset
|
||||
- [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon
|
||||
- [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API
|
||||
- [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node
|
||||
- [Content blocking](/docs/content-blocking.md) support for operators of public nodes
|
||||
|
||||
### Other implementations
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import (
|
||||
cid "github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipfs/kubo/core/commands/cmdenv"
|
||||
"github.com/ipfs/kubo/core/commands/cmdutils"
|
||||
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
gocar "github.com/ipld/go-car"
|
||||
@ -21,12 +22,10 @@ import (
|
||||
)
|
||||
|
||||
func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
||||
c, err := cid.Decode(req.Arguments[0])
|
||||
// Accept CID or a content path
|
||||
p, err := cmdutils.PathOrCidPath(req.Arguments[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"unable to parse root specification (currently only bare CIDs are supported): %s",
|
||||
err,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
api, err := cmdenv.GetApi(env, req)
|
||||
@ -34,6 +33,13 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
return err
|
||||
}
|
||||
|
||||
// Resolve path and confirm the root block is available, fail fast if not
|
||||
b, err := api.Block().Stat(req.Context, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := b.Path().RootCid()
|
||||
|
||||
pipeR, pipeW := io.Pipe()
|
||||
|
||||
errCh := make(chan error, 2) // we only report the 1st error
|
||||
|
||||
58
core/core.go
58
core/core.go
@ -76,35 +76,39 @@ type IpfsNode struct {
|
||||
PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
|
||||
|
||||
// Services
|
||||
Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
|
||||
Blockstore bstore.GCBlockstore // the block store (lower level)
|
||||
Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
|
||||
BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
|
||||
GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
|
||||
Blocks bserv.BlockService // the block service, get/add blocks.
|
||||
DAG ipld.DAGService // the merkle dag service, get/add objects.
|
||||
IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
|
||||
UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
|
||||
Reporter *metrics.BandwidthCounter `optional:"true"`
|
||||
Discovery mdns.Service `optional:"true"`
|
||||
FilesRoot *mfs.Root
|
||||
RecordValidator record.Validator
|
||||
Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
|
||||
Blockstore bstore.GCBlockstore // the block store (lower level)
|
||||
Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
|
||||
BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
|
||||
GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
|
||||
Blocks bserv.BlockService // the block service, get/add blocks.
|
||||
DAG ipld.DAGService // the merkle dag service, get/add objects.
|
||||
IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
|
||||
UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
|
||||
OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks
|
||||
OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks
|
||||
Reporter *metrics.BandwidthCounter `optional:"true"`
|
||||
Discovery mdns.Service `optional:"true"`
|
||||
FilesRoot *mfs.Root
|
||||
RecordValidator record.Validator
|
||||
|
||||
// Online
|
||||
PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
|
||||
Peering *peering.PeeringService `optional:"true"`
|
||||
Filters *ma.Filters `optional:"true"`
|
||||
Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
|
||||
Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
|
||||
DNSResolver *madns.Resolver // the DNS resolver
|
||||
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
|
||||
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
|
||||
Exchange exchange.Interface // the block exchange + strategy (bitswap)
|
||||
Namesys namesys.NameSystem // the name system, resolves paths to hashes
|
||||
Provider provider.System // the value provider system
|
||||
IpnsRepub *ipnsrp.Republisher `optional:"true"`
|
||||
GraphExchange graphsync.GraphExchange `optional:"true"`
|
||||
ResourceManager network.ResourceManager `optional:"true"`
|
||||
PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
|
||||
Peering *peering.PeeringService `optional:"true"`
|
||||
Filters *ma.Filters `optional:"true"`
|
||||
Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
|
||||
Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
|
||||
DNSResolver *madns.Resolver // the DNS resolver
|
||||
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
|
||||
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
|
||||
OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
|
||||
OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
|
||||
Exchange exchange.Interface // the block exchange + strategy (bitswap)
|
||||
Namesys namesys.NameSystem // the name system, resolves paths to hashes
|
||||
Provider provider.System // the value provider system
|
||||
IpnsRepub *ipnsrp.Republisher `optional:"true"`
|
||||
GraphExchange graphsync.GraphExchange `optional:"true"`
|
||||
ResourceManager network.ResourceManager `optional:"true"`
|
||||
|
||||
PubSub *pubsub.PubSub `optional:"true"`
|
||||
PSRouter *psrouter.PubsubValueStore `optional:"true"`
|
||||
|
||||
@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption {
|
||||
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
|
||||
bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
|
||||
|
||||
backend, err := gateway.NewBlocksBackend(bserv)
|
||||
backend, err := gateway.NewBlocksBackend(bserv,
|
||||
// GatewayOverLibp2p only returns things that are in local blockstore
|
||||
// (same as Gateway.NoFetch=true), we have to pass offline path resolver
|
||||
gateway.WithResolver(n.OfflineUnixFSPathResolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -111,6 +115,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
|
||||
bserv := n.Blocks
|
||||
var vsRouting routing.ValueStore = n.Routing
|
||||
nsys := n.Namesys
|
||||
pathResolver := n.UnixFSPathResolver
|
||||
|
||||
if cfg.Gateway.NoFetch {
|
||||
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
|
||||
|
||||
@ -130,9 +136,17 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error constructing namesys: %w", err)
|
||||
}
|
||||
|
||||
// Gateway.NoFetch=true requires offline path resolver
|
||||
// to avoid fetching missing blocks during path traversal
|
||||
pathResolver = n.OfflineUnixFSPathResolver
|
||||
}
|
||||
|
||||
backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
|
||||
backend, err := gateway.NewBlocksBackend(bserv,
|
||||
gateway.WithValueStore(vsRouting),
|
||||
gateway.WithNameSystem(nsys),
|
||||
gateway.WithResolver(pathResolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"github.com/ipfs/boxo/blockservice"
|
||||
blockstore "github.com/ipfs/boxo/blockstore"
|
||||
exchange "github.com/ipfs/boxo/exchange"
|
||||
offline "github.com/ipfs/boxo/exchange/offline"
|
||||
"github.com/ipfs/boxo/fetcher"
|
||||
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
|
||||
"github.com/ipfs/boxo/filestore"
|
||||
@ -21,9 +22,6 @@ import (
|
||||
format "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipfs/go-unixfsnode"
|
||||
dagpb "github.com/ipld/go-codec-dagpb"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
basicnode "github.com/ipld/go-ipld-prime/node/basic"
|
||||
"github.com/ipld/go-ipld-prime/schema"
|
||||
"go.uber.org/fx"
|
||||
|
||||
"github.com/ipfs/kubo/core/node/helpers"
|
||||
@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter {
|
||||
// FetchersOut allows injection of fetchers.
|
||||
type FetchersOut struct {
|
||||
fx.Out
|
||||
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
|
||||
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
|
||||
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
|
||||
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
|
||||
OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
|
||||
OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
|
||||
}
|
||||
|
||||
// FetchersIn allows using fetchers for other dependencies.
|
||||
type FetchersIn struct {
|
||||
fx.In
|
||||
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
|
||||
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
|
||||
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
|
||||
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
|
||||
OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
|
||||
OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
|
||||
}
|
||||
|
||||
// FetcherConfig returns a fetcher config that can build new fetcher instances
|
||||
func FetcherConfig(bs blockservice.BlockService) FetchersOut {
|
||||
ipldFetcher := bsfetcher.NewFetcherConfig(bs)
|
||||
ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
|
||||
if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
|
||||
return tlnkNd.LinkTargetNodePrototype(), nil
|
||||
}
|
||||
return basicnode.Prototype.Any, nil
|
||||
})
|
||||
|
||||
ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
|
||||
unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
|
||||
return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher}
|
||||
|
||||
// Construct offline versions which we can safely use in contexts where
|
||||
// path resolution should not fetch new blocks via exchange.
|
||||
offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
|
||||
offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
|
||||
offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
|
||||
offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
|
||||
|
||||
return FetchersOut{
|
||||
IPLDFetcher: ipldFetcher,
|
||||
UnixfsFetcher: unixFSFetcher,
|
||||
OfflineIPLDFetcher: offlineIpldFetcher,
|
||||
OfflineUnixfsFetcher: offlineUnixFSFetcher,
|
||||
}
|
||||
}
|
||||
|
||||
// PathResolversOut allows injection of path resolvers
|
||||
type PathResolversOut struct {
|
||||
fx.Out
|
||||
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
|
||||
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
|
||||
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
|
||||
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
|
||||
OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"`
|
||||
OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"`
|
||||
}
|
||||
|
||||
// PathResolverConfig creates path resolvers with the given fetchers.
|
||||
func PathResolverConfig(fetchers FetchersIn) PathResolversOut {
|
||||
return PathResolversOut{
|
||||
IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
|
||||
UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
|
||||
IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
|
||||
UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
|
||||
OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher),
|
||||
OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
- [Overview](#overview)
|
||||
- [🔦 Highlights](#-highlights)
|
||||
- [Support for content blocking](#support-for-content-blocking)
|
||||
- [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful)
|
||||
- [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults)
|
||||
- [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching)
|
||||
@ -16,6 +17,14 @@
|
||||
|
||||
### 🔦 Highlights
|
||||
|
||||
#### Support for content blocking
|
||||
|
||||
This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/).
|
||||
Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
|
||||
The official build does not ship with any denylists.
|
||||
|
||||
Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md)
|
||||
|
||||
#### Gateway: the root of the CARs are no longer meaningful
|
||||
|
||||
When requesting a CAR from the gateway, the root of the CAR might no longer be
|
||||
|
||||
73
docs/content-blocking.md
Normal file
73
docs/content-blocking.md
Normal file
@ -0,0 +1,73 @@
|
||||
<h1 align="center">
|
||||
<br>
|
||||
<a href="#readme"><img src="https://github.com/ipfs-shipyard/nopfs/blob/41484a818e6542314f784da852fc41b76f2d48a6/logo.png?raw=true" alt="content blocking logo" title="content blocking in Kubo" width="200"></a>
|
||||
<br>
|
||||
Content Blocking in Kubo
|
||||
<br>
|
||||
</h1>
|
||||
|
||||
Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
|
||||
|
||||
## Default behavior
|
||||
|
||||
Official Kubo build does not ship with any denylists enabled by default.
|
||||
|
||||
Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
|
||||
|
||||
## How to enable blocking
|
||||
|
||||
Place a `*.deny` file in one of directories:
|
||||
|
||||
- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set)
|
||||
- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set)
|
||||
- `/etc/ipfs/denylists/` (global)
|
||||
|
||||
Files need to be present before starting the `ipfs daemon` in order to be watched for updates.
|
||||
|
||||
If a new denylist file is added, `ipfs daemon` needs to be restarted.
|
||||
|
||||
CLI and Gateway users will receive errors in response to request impacted by a blocklist:
|
||||
|
||||
```
|
||||
Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided
|
||||
```
|
||||
|
||||
End user is not informed about the exact reason, see [How to
|
||||
debug](#how-to-debug) if you need to find out which line of which denylist
|
||||
caused the request to be blocked.
|
||||
|
||||
## Denylist file format
|
||||
|
||||
[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
|
||||
|
||||
Clear-text rules are simple: just put content paths to block, one per line.
|
||||
Paths with unicode and whitespace need to be percend-encoded:
|
||||
|
||||
```
|
||||
/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
|
||||
/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png
|
||||
```
|
||||
|
||||
Sensitive content paths can be double-hashed to block without revealing them.
|
||||
Double-hashed list example: https://badbits.dwebops.pub/badbits.deny
|
||||
|
||||
See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples.
|
||||
|
||||
## How to suspend blocking without removing denylists
|
||||
|
||||
Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon.
|
||||
|
||||
|
||||
## How to debug
|
||||
|
||||
Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`
|
||||
|
||||
All block events are logged as warnings on a separate level named `nopfs-blocks`.
|
||||
|
||||
To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`:
|
||||
|
||||
```
|
||||
WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9)
|
||||
```
|
||||
|
||||
|
||||
@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for
|
||||
inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
|
||||
or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
|
||||
|
||||
|
||||
## `IPFS_CONTENT_BLOCKING_DISABLE`
|
||||
|
||||
Disables the content-blocking subsystem. No denylists will be watched and no
|
||||
content will be blocked.
|
||||
|
||||
## `LIBP2P_TCP_REUSEPORT`
|
||||
|
||||
Kubo tries to reuse the same source port for all connections to improve NAT
|
||||
|
||||
@ -7,7 +7,7 @@ go 1.20
|
||||
replace github.com/ipfs/kubo => ./../../..
|
||||
|
||||
require (
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
|
||||
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
|
||||
github.com/libp2p/go-libp2p v0.31.0
|
||||
github.com/multiformats/go-multiaddr v0.11.0
|
||||
@ -40,6 +40,7 @@ require (
|
||||
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
|
||||
github.com/go-logr/logr v1.2.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
@ -60,6 +61,8 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/huin/goupnp v1.2.0 // indirect
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect
|
||||
github.com/ipfs/bbloom v0.0.4 // indirect
|
||||
github.com/ipfs/go-bitfield v1.1.0 // indirect
|
||||
github.com/ipfs/go-block-format v0.2.0 // indirect
|
||||
@ -194,5 +197,6 @@ require (
|
||||
google.golang.org/grpc v1.55.0 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.2.1 // indirect
|
||||
)
|
||||
|
||||
@ -163,6 +163,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
@ -298,10 +299,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
|
||||
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
@ -1001,6 +1006,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
|
||||
4
go.mod
4
go.mod
@ -15,7 +15,9 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.6.0
|
||||
github.com/google/uuid v1.3.1
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
|
||||
github.com/ipfs/go-block-format v0.2.0
|
||||
github.com/ipfs/go-cid v0.4.1
|
||||
github.com/ipfs/go-cidutil v0.1.0
|
||||
|
||||
8
go.sum
8
go.sum
@ -333,10 +333,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
|
||||
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
|
||||
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
|
||||
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest"
|
||||
pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
|
||||
pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
|
||||
pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
|
||||
pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
|
||||
)
|
||||
|
||||
@ -22,4 +23,5 @@ func init() {
|
||||
Preload(pluginlevelds.Plugins...)
|
||||
Preload(pluginpeerlog.Plugins...)
|
||||
Preload(pluginfxtest.Plugins...)
|
||||
Preload(pluginnopfs.Plugins...)
|
||||
}
|
||||
|
||||
@ -11,3 +11,4 @@ flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
|
||||
levelds github.com/ipfs/kubo/plugin/plugins/levelds *
|
||||
peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
|
||||
fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
|
||||
nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
|
||||
85
plugin/plugins/nopfs/nopfs.go
Normal file
85
plugin/plugins/nopfs/nopfs.go
Normal file
@ -0,0 +1,85 @@
|
||||
package nopfs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/ipfs-shipyard/nopfs"
|
||||
"github.com/ipfs-shipyard/nopfs/ipfs"
|
||||
"github.com/ipfs/kubo/config"
|
||||
"github.com/ipfs/kubo/core"
|
||||
"github.com/ipfs/kubo/core/node"
|
||||
"github.com/ipfs/kubo/plugin"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Plugins sets the list of plugins to be loaded.
|
||||
var Plugins = []plugin.Plugin{
|
||||
&nopfsPlugin{},
|
||||
}
|
||||
|
||||
// fxtestPlugin is used for testing the fx plugin.
|
||||
// It merely adds an fx option that logs a debug statement, so we can verify that it works in tests.
|
||||
type nopfsPlugin struct{}
|
||||
|
||||
var _ plugin.PluginFx = (*nopfsPlugin)(nil)
|
||||
|
||||
func (p *nopfsPlugin) Name() string {
|
||||
return "nopfs"
|
||||
}
|
||||
|
||||
func (p *nopfsPlugin) Version() string {
|
||||
return "0.0.10"
|
||||
}
|
||||
|
||||
func (p *nopfsPlugin) Init(env *plugin.Environment) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MakeBlocker is a factory for the blocker so that it can be provided with Fx.
|
||||
func MakeBlocker() (*nopfs.Blocker, error) {
|
||||
ipfsPath, err := config.PathRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defaultFiles, err := nopfs.GetDenylistFiles()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kuboFiles, err := nopfs.GetDenylistFilesInDir(filepath.Join(ipfsPath, "denylists"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files := append(defaultFiles, kuboFiles...)
|
||||
|
||||
return nopfs.NewBlocker(files)
|
||||
}
|
||||
|
||||
// PathResolvers returns wrapped PathResolvers for Kubo.
|
||||
func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut {
|
||||
res := node.PathResolverConfig(fetchers)
|
||||
return node.PathResolversOut{
|
||||
IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker),
|
||||
UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker),
|
||||
OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker),
|
||||
OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *nopfsPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) {
|
||||
if os.Getenv("IPFS_CONTENT_BLOCKING_DISABLE") != "" {
|
||||
return info.FXOptions, nil
|
||||
}
|
||||
|
||||
opts := append(
|
||||
info.FXOptions,
|
||||
fx.Provide(MakeBlocker),
|
||||
fx.Decorate(ipfs.WrapBlockService),
|
||||
fx.Decorate(ipfs.WrapNameSystem),
|
||||
fx.Decorate(PathResolvers),
|
||||
)
|
||||
return opts, nil
|
||||
}
|
||||
303
test/cli/content_blocking_test.go
Normal file
303
test/cli/content_blocking_test.go
Normal file
@ -0,0 +1,303 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/libp2p/go-libp2p"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
libp2phttp "github.com/libp2p/go-libp2p/p2p/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestContentBlocking(t *testing.T) {
|
||||
// NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP
|
||||
// and running in parallel could impact other tests
|
||||
|
||||
const blockedMsg = "blocked and cannot be provided"
|
||||
const statusExpl = "specific HTTP error code is expected"
|
||||
const bodyExpl = "Error message informing about content block is expected"
|
||||
|
||||
h := harness.NewT(t)
|
||||
|
||||
// Init IPFS_PATH
|
||||
node := h.NewNode().Init("--empty-repo", "--profile=test")
|
||||
|
||||
// Create CIDs we use in test
|
||||
h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content")
|
||||
parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed()
|
||||
|
||||
h.WriteFile("directly-blocked-file.txt", "directly blocked file content")
|
||||
blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed()
|
||||
|
||||
h.WriteFile("not-blocked-file.txt", "not blocked file content")
|
||||
allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed()
|
||||
|
||||
// Create denylist at $IPFS_PATH/denylists/test.deny
|
||||
denylistTmp := h.WriteToTemp("name: test list\n---\n" +
|
||||
"//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR))
|
||||
"//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path))
|
||||
"//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/)
|
||||
"//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath)
|
||||
"/ipfs/" + blockedCID + "\n" + // block specific CID
|
||||
"/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath
|
||||
"/ipns/blocked-cid.example.com\n" +
|
||||
"/ipns/blocked-dnslink.example.com\n")
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil {
|
||||
log.Panicf("failed to create denylists dir: %s", err.Error())
|
||||
}
|
||||
if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil {
|
||||
log.Panicf("failed to create test denylist: %s", err.Error())
|
||||
}
|
||||
|
||||
// Add two entries to namesys resolution cache
|
||||
// /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution)
|
||||
// /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths)
|
||||
os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
|
||||
defer os.Unsetenv("IPFS_NS_MAP")
|
||||
|
||||
// Enable GatewayOverLibp2p as we want to test denylist there too
|
||||
node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true")
|
||||
|
||||
// Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny
|
||||
node.StartDaemon() // we need online mode for GatewayOverLibp2p tests
|
||||
client := node.GatewayClient()
|
||||
|
||||
// First, confirm gateway works
|
||||
t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
resp := client.Get("/ipfs/" + allowedCID)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, "not blocked file content", resp.Body)
|
||||
})
|
||||
|
||||
// Then, does the most basic blocking case work?
|
||||
t.Run("Gateway Denies directly blocked CID", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
resp := client.Get("/ipfs/" + blockedCID)
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.NotEqual(t, "directly blocked file content", resp.Body)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
// Confirm parent of blocked subpath is not blocked
|
||||
t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
resp := client.Get("/ipfs/" + parentDirCID)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
// Ok, now the full list of test cases we want to cover in both CLI and Gateway
|
||||
testCases := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{
|
||||
name: "directly blocked CID",
|
||||
path: "/ipfs/" + blockedCID,
|
||||
},
|
||||
{
|
||||
name: "indirectly blocked file (on a blocked subpath)",
|
||||
path: "/ipfs/" + parentDirCID + "/subdir/indirectly-blocked-file.txt",
|
||||
},
|
||||
{
|
||||
name: "/ipns path that resolves to a blocked CID",
|
||||
path: "/ipns/blocked-cid.example.com",
|
||||
},
|
||||
{
|
||||
name: "/ipns Path that is blocked by DNSLink name",
|
||||
path: "/ipns/blocked-dnslink.example.com",
|
||||
},
|
||||
{
|
||||
name: "double-hash CID block (sha256-multihash)",
|
||||
path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR",
|
||||
},
|
||||
{
|
||||
name: "double-hash Path block (blake3-multihash)",
|
||||
path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path",
|
||||
},
|
||||
{
|
||||
name: "legacy CID double-hash block (sha256)",
|
||||
path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq",
|
||||
},
|
||||
|
||||
{
|
||||
name: "legacy Path double-hash block (sha256)",
|
||||
path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath",
|
||||
},
|
||||
}
|
||||
|
||||
// Which specific cliCmds we test against testCases
|
||||
cliCmds := [][]string{
|
||||
{"block", "get"},
|
||||
{"block", "stat"},
|
||||
{"dag", "get"},
|
||||
{"dag", "export"},
|
||||
{"dag", "stat"},
|
||||
{"cat"},
|
||||
{"ls"},
|
||||
{"get"},
|
||||
{"refs"},
|
||||
}
|
||||
|
||||
expectedMsg := blockedMsg
|
||||
for _, testCase := range testCases {
|
||||
|
||||
// Confirm that denylist is active for every command in 'cliCmds' x 'testCases'
|
||||
for _, cmd := range cliCmds {
|
||||
cmd := cmd
|
||||
cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name)
|
||||
t.Run(cliTestName, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
args := append(cmd, testCase.path)
|
||||
errMsg := node.RunIPFS(args...).Stderr.Trimmed()
|
||||
if !strings.Contains(errMsg, expectedMsg) {
|
||||
t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, errMsg)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Confirm that denylist is active for every content path in 'testCases'
|
||||
gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name)
|
||||
t.Run(gwTestName, func(t *testing.T) {
|
||||
resp := client.Get(testCase.path)
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Extra edge cases on subdomain gateway
|
||||
|
||||
t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gwURL, _ := url.Parse(node.GatewayURL())
|
||||
resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) {
|
||||
r.Host = "localhost:" + gwURL.Port()
|
||||
})
|
||||
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gwURL, _ := url.Parse(node.GatewayURL())
|
||||
resp := client.Get("/", func(r *http.Request) {
|
||||
r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port()
|
||||
})
|
||||
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gwURL, _ := url.Parse(node.GatewayURL())
|
||||
resp := client.Get("/", func(r *http.Request) {
|
||||
// Inlined DNSLink to fit in single DNS label for TLS interop:
|
||||
// https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header
|
||||
r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port()
|
||||
})
|
||||
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
// We need to confirm denylist is active when gateway is run in NoFetch
|
||||
// mode (which usually swaps blockservice to a read-only one, and that swap
|
||||
// may cause denylists to not be applied, as it is a separate code path)
|
||||
t.Run("GatewayNoFetch", func(t *testing.T) {
|
||||
// NOTE: we don't run this in parallel, as it requires restart with different config
|
||||
|
||||
// Switch gateway to NoFetch mode
|
||||
node.StopDaemon()
|
||||
node.IPFS("config", "--json", "Gateway.NoFetch", "true")
|
||||
node.StartDaemon()
|
||||
|
||||
// update client, as the port of test node might've changed after restart
|
||||
client = node.GatewayClient()
|
||||
|
||||
// First, confirm gateway works
|
||||
t.Run("Allows CID that is not blocked", func(t *testing.T) {
|
||||
resp := client.Get("/ipfs/" + allowedCID)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, "not blocked file content", resp.Body)
|
||||
})
|
||||
|
||||
// Then, does the most basic blocking case work?
|
||||
t.Run("Denies directly blocked CID", func(t *testing.T) {
|
||||
resp := client.Get("/ipfs/" + blockedCID)
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
assert.NotEqual(t, "directly blocked file content", resp.Body)
|
||||
assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
|
||||
})
|
||||
|
||||
// Restore default
|
||||
node.StopDaemon()
|
||||
node.IPFS("config", "--json", "Gateway.NoFetch", "false")
|
||||
node.StartDaemon()
|
||||
client = node.GatewayClient()
|
||||
})
|
||||
|
||||
// We need to confirm denylist is active on the
|
||||
// trustless gateway exposed over libp2p
|
||||
// when Experimental.GatewayOverLibp2p=true
|
||||
// (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p)
|
||||
// NOTE: this type fo gateway is hardcoded to be NoFetch: it does not fetch
|
||||
// data that is not in local store, so we only need to run it once: a
|
||||
// simple smoke-test for allowed CID and blockedCID.
|
||||
t.Run("GatewayOverLibp2p", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Create libp2p client that connects to our node over
|
||||
// /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol
|
||||
clientHost, err := libp2p.New(libp2p.NoListenAddrs)
|
||||
require.NoError(t, err)
|
||||
err = clientHost.Connect(context.Background(), peer.AddrInfo{
|
||||
ID: node.PeerID(),
|
||||
Addrs: node.SwarmAddrs(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("Serves Allowed CID", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, string(body), "not blocked file content", bodyExpl)
|
||||
})
|
||||
|
||||
t.Run("Denies Blocked CID", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, string(body), "directly blocked file content")
|
||||
assert.Contains(t, string(body), blockedMsg, bodyExpl)
|
||||
})
|
||||
})
|
||||
}
|
||||
@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../
|
||||
require (
|
||||
github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
|
||||
github.com/golangci/golangci-lint v1.54.1
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
|
||||
github.com/ipfs/go-cid v0.4.1
|
||||
github.com/ipfs/go-cidutil v0.1.0
|
||||
github.com/ipfs/go-datastore v0.6.0
|
||||
|
||||
@ -398,8 +398,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
|
||||
github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
|
||||
|
||||
@ -178,13 +178,11 @@ test_expect_success "basic offline export of 'getting started' dag works" '
|
||||
ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null
|
||||
'
|
||||
|
||||
|
||||
echo "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
|
||||
test_expect_success "basic offline export of nonexistent cid" '
|
||||
! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null
|
||||
'
|
||||
test_expect_success "correct error" '
|
||||
test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
|
||||
test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual
|
||||
'
|
||||
|
||||
cat >multiroot_import_json_stats_expected <<EOE
|
||||
|
||||
Loading…
Reference in New Issue
Block a user