feat(gateway): support for IPIP-402 CAR params (#9914)

This commit is contained in:
Henrique Dias 2023-06-08 17:32:18 +02:00 committed by GitHub
parent 30f5e54e19
commit 2716cd987f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 48 additions and 363 deletions

View File

@ -28,24 +28,21 @@ import (
func GatewayOption(paths ...string) ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
gwConfig, err := getGatewayConfig(n)
config, err := getGatewayConfig(n)
if err != nil {
return nil, err
}
gwAPI, err := newGatewayBackend(n)
backend, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
gw := gateway.NewHandler(gwConfig, gwAPI)
gw = otelhttp.NewHandler(gw, "Gateway")
// By default, our HTTP handler is the gateway handler.
handler := gw.ServeHTTP
handler := gateway.NewHandler(config, backend)
handler = otelhttp.NewHandler(handler, "Gateway")
for _, p := range paths {
mux.HandleFunc(p+"/", handler)
mux.HandleFunc(p+"/", handler.ServeHTTP)
}
return mux, nil
@ -54,18 +51,18 @@ func GatewayOption(paths ...string) ServeOption {
func HostnameOption() ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
gwConfig, err := getGatewayConfig(n)
config, err := getGatewayConfig(n)
if err != nil {
return nil, err
}
gwAPI, err := newGatewayBackend(n)
backend, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
childMux := http.NewServeMux()
mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP)
return childMux, nil
}
}
@ -111,11 +108,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
}
}
gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
if err != nil {
return nil, err
}
return &offlineGatewayErrWrapper{gwimpl: gw}, nil
return &offlineGatewayErrWrapper{gwimpl: backend}, nil
}
type offlineGatewayErrWrapper struct {
@ -159,10 +156,10 @@ func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway
return md, err
}
func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) {
md, data, err := o.gwimpl.GetCAR(ctx, path, params)
err = offlineErrWrap(err)
return md, data, errCh, err
return md, data, err
}
func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
@ -191,12 +188,12 @@ var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
var subdomainGatewaySpec = &gateway.Specification{
var subdomainGatewaySpec = &gateway.PublicGateway{
Paths: defaultPaths,
UseSubdomains: true,
}
var defaultKnownGateways = map[string]*gateway.Specification{
var defaultKnownGateways = map[string]*gateway.PublicGateway{
"localhost": subdomainGatewaySpec,
}
@ -218,7 +215,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
Headers: headers,
DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
NoDNSLink: cfg.Gateway.NoDNSLink,
PublicGateways: map[string]*gateway.Specification{},
PublicGateways: map[string]*gateway.PublicGateway{},
}
// Add default implicit known gateways, such as subdomain gateway on localhost.
@ -235,7 +232,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
continue
}
gwCfg.PublicGateways[hostname] = &gateway.Specification{
gwCfg.PublicGateways[hostname] = &gateway.PublicGateway{
Paths: gw.Paths,
NoDNSLink: gw.NoDNSLink,
UseSubdomains: gw.UseSubdomains,

View File

@ -104,6 +104,10 @@ $ curl "https://subdomain-gw.example.net/ipfs/${cid}/"
Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913).
#### Gateway: support for CAR parameters of IPIP-402
The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402).
#### `ipfs dag stat` deduping statistics
`ipfs dat stat` now accept multiple CIDs and will dump advanced statistics

View File

@ -7,7 +7,7 @@ go 1.18
replace github.com/ipfs/kubo => ./../../..
require (
github.com/ipfs/boxo v0.9.0
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.27.5
github.com/multiformats/go-multiaddr v0.9.0
@ -87,7 +87,7 @@ require (
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.6.0 // indirect
github.com/ipfs/go-unixfsnode v1.7.1 // indirect
github.com/ipld/edelweiss v0.2.0 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.20.0 // indirect
@ -139,6 +139,7 @@ require (
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/openzipkin/zipkin-go v0.4.1 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
@ -156,6 +157,7 @@ require (
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect

View File

@ -319,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@ -409,8 +409,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
@ -421,6 +421,7 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt
github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
@ -646,6 +647,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -781,6 +783,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=

6
go.mod
View File

@ -16,7 +16,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/boxo v0.9.0
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0
@ -37,7 +37,7 @@ require (
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-metrics-interface v0.0.1
github.com/ipfs/go-metrics-prometheus v0.0.2
github.com/ipfs/go-unixfsnode v1.6.0
github.com/ipfs/go-unixfsnode v1.7.1
github.com/ipld/go-codec-dagpb v1.6.0
github.com/ipld/go-ipld-prime v0.20.0
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
@ -177,6 +177,7 @@ require (
github.com/onsi/ginkgo/v2 v2.9.2 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/openzipkin/zipkin-go v0.4.1 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
@ -197,6 +198,7 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect

10
go.sum
View File

@ -354,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@ -448,8 +448,8 @@ github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnz
github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
@ -715,6 +715,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -894,6 +895,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=

View File

@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../
require (
github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
github.com/golangci/golangci-lint v1.49.0
github.com/ipfs/boxo v0.9.0
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0
github.com/ipfs/go-datastore v0.6.0
@ -131,7 +131,7 @@ require (
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.6.0 // indirect
github.com/ipfs/go-unixfsnode v1.7.1 // indirect
github.com/ipfs/kubo v0.16.0 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect

View File

@ -412,8 +412,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=
@ -465,8 +465,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipfs/hang-fds v0.1.0 h1:deBiFlWHsVGzJ0ZMaqscEqRM1r2O1rFZ59UiQXb1Xko=
github.com/ipfs/hang-fds v0.1.0/go.mod h1:29VLWOn3ftAgNNgXg/al7b11UzuQ+w7AwtCGcTaWkbM=

View File

@ -1,136 +0,0 @@
#!/usr/bin/env bash
test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support"
. lib/test-lib.sh
test_init_ipfs
test_launch_ipfs_daemon_without_network
# CAR stream is not deterministic, as blocks can arrive in random order,
# but if we have a small file that fits into a single block, and export its CID
# we will get a CAR that is a deterministic array of bytes.
# Import test case
# See the static fixtures in ./t0118-gateway-car/
test_expect_success "Add the dir test directory" '
cp ../t0118-gateway-car/test-dag.car ./test-dag.car &&
cp ../t0118-gateway-car/deterministic.car ./deterministic.car
'
ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
# GET a reference DAG with dag-cbor+dag-pb+raw blocks as CAR
# This test uses official CARv1 fixture from https://ipld.io/specs/transport/car/fixture/carv1-basic/
test_expect_success "GET for application/vnd.ipld.car with dag-cbor root returns a CARv1 stream with full DAG" '
ipfs dag import ../t0118-gateway-car/carv1-basic.car &&
DAG_CBOR_CID=bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm &&
curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_CID" -o gateway-dag-cbor.car &&
purge_blockstore &&
ipfs dag import gateway-dag-cbor.car &&
ipfs dag stat --offline $DAG_CBOR_CID
'
# GET unixfs file as CAR
# (by using a single file we ensure deterministic result that can be compared byte-for-byte)
test_expect_success "GET with format=car param returns a CARv1 stream" '
ipfs dag import test-dag.car &&
curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car &&
test_cmp deterministic.car gateway-param.car
'
test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" '
ipfs dag import test-dag.car &&
curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car &&
test_cmp deterministic.car gateway-header.car
'
# explicit version=1
test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" '
ipfs dag import test-dag.car &&
curl -sX GET -H "Accept: application/vnd.ipld.car;version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
test_cmp deterministic.car gateway-header-v1.car
'
# explicit version=1 with whitepace
test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream (with whitespace)" '
ipfs dag import test-dag.car &&
curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
test_cmp deterministic.car gateway-header-v1.car
'
# explicit version=2
test_expect_success "GET for application/vnd.ipld.raw version=2 returns HTTP 400 Bad Request error" '
curl -svX GET -H "Accept: application/vnd.ipld.car;version=2" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" > curl_output 2>&1 &&
cat curl_output &&
grep "400 Bad Request" curl_output &&
grep "unsupported CAR version" curl_output
'
# GET unixfs directory as a CAR with DAG and some selector
# TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769
test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" '
ipfs dag import test-dag.car &&
curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car &&
purge_blockstore &&
ipfs dag import gateway-dir.car &&
ipfs dag stat --offline $ROOT_DIR_CID
'
# Make sure expected HTTP headers are returned with the CAR bytes
test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" '
ipfs dag import test-dag.car &&
curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output &&
cat curl_output &&
grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output
'
# CAR is streamed, gateway may not have the entire thing, unable to calculate total size
test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" '
grep -qv "< Content-Length:" curl_output
'
test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" '
grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output
'
test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" '
grep "< X-Content-Type-Options: nosniff" curl_output
'
# CAR is streamed, gateway may not have the entire thing, unable to support range-requests
# Partial downloads and resumes should be handled using
# IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
test_expect_success "GET response for application/vnd.ipld.car includes Accept-Ranges header" '
grep "< Accept-Ranges: none" curl_output
'
test_expect_success "GET for application/vnd.ipld.car with query filename includes Content-Disposition with custom filename" '
curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?filename=foobar.car" >/dev/null 2>curl_output_filename &&
cat curl_output_filename &&
grep "< Content-Disposition: attachment\; filename=\"foobar.car\"" curl_output_filename
'
# Cache control HTTP headers
test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" '
grep "< Etag: W/\"${FILE_CID}.car\"" curl_output
'
# (basic checks, detailed behavior for some fields is tested in t0116-gateway-cache.sh)
test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" '
grep "< X-Ipfs-Path" curl_output &&
grep "< X-Ipfs-Roots" curl_output
'
test_expect_success "GET response for application/vnd.ipld.car includes same Cache-Control as a block or a file" '
grep "< Cache-Control: public, max-age=29030400, immutable" curl_output
'
test_kill_ipfs_daemon
test_done

View File

@ -1,30 +0,0 @@
# Dataset description/sources
- carv1-basic.car
- raw CARv1
- Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.car
- carv1-basic.json
- description of the contents and layout of the raw CAR, encoded in DAG-JSON
- Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.json
- test-dag.car + deterministic.car
- raw CARv1
generated with:
```sh
# using ipfs version 0.18.1
mkdir -p subdir &&
echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
ipfs dag export $ROOT_DIR_CID > test-dag.car &&
ipfs dag export $FILE_CID > deterministic.car &&
echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./
echo FILE_CID=${FILE_CID} # /\subdir/ascii.txt
# ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
# FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
```

View File

@ -1,159 +0,0 @@
{
"blocks": [
{
"blockLength": 55,
"blockOffset": 137,
"cid": {
"/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
},
"content": {
"link": {
"/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
},
"name": "blip"
},
"length": 92,
"offset": 100
},
{
"blockLength": 97,
"blockOffset": 228,
"cid": {
"/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
},
"content": {
"Links": [
{
"Hash": {
"/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
},
"Name": "bear",
"Tsize": 4
},
{
"Hash": {
"/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
},
"Name": "second",
"Tsize": 149
}
]
},
"length": 133,
"offset": 192
},
{
"blockLength": 4,
"blockOffset": 362,
"cid": {
"/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
},
"content": {
"/": {
"bytes": "Y2NjYw"
}
},
"length": 41,
"offset": 325
},
{
"blockLength": 94,
"blockOffset": 402,
"cid": {
"/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
},
"content": {
"Links": [
{
"Hash": {
"/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
},
"Name": "dog",
"Tsize": 4
},
{
"Hash": {
"/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
},
"Name": "first",
"Tsize": 51
}
]
},
"length": 130,
"offset": 366
},
{
"blockLength": 4,
"blockOffset": 533,
"cid": {
"/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
},
"content": {
"/": {
"bytes": "YmJiYg"
}
},
"length": 41,
"offset": 496
},
{
"blockLength": 47,
"blockOffset": 572,
"cid": {
"/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
},
"content": {
"Links": [
{
"Hash": {
"/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
},
"Name": "cat",
"Tsize": 4
}
]
},
"length": 82,
"offset": 537
},
{
"blockLength": 4,
"blockOffset": 656,
"cid": {
"/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
},
"content": {
"/": {
"bytes": "YWFhYQ"
}
},
"length": 41,
"offset": 619
},
{
"blockLength": 18,
"blockOffset": 697,
"cid": {
"/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
},
"content": {
"link": null,
"name": "limbo"
},
"length": 55,
"offset": 660
}
],
"header": {
"roots": [
{
"/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
},
{
"/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
}
],
"version": 1
}
}