From 2716cd987fcf9ee3c6c1ffd0d5cc68283c32e557 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 8 Jun 2023 17:32:18 +0200 Subject: [PATCH] feat(gateway): support for IPIP-402 CAR params (#9914) --- core/corehttp/gateway.go | 37 ++-- docs/changelogs/v0.21.md | 4 + docs/examples/kubo-as-a-library/go.mod | 6 +- docs/examples/kubo-as-a-library/go.sum | 11 +- go.mod | 6 +- go.sum | 10 +- test/dependencies/go.mod | 4 +- test/dependencies/go.sum | 8 +- test/sharness/t0118-gateway-car.sh | 136 --------------- test/sharness/t0118-gateway-car/README.md | 30 ---- .../t0118-gateway-car/carv1-basic.car | Bin 715 -> 0 bytes .../t0118-gateway-car/carv1-basic.json | 159 ------------------ .../t0118-gateway-car/deterministic.car | Bin 127 -> 0 bytes test/sharness/t0118-gateway-car/test-dag.car | Bin 312 -> 0 bytes 14 files changed, 48 insertions(+), 363 deletions(-) delete mode 100755 test/sharness/t0118-gateway-car.sh delete mode 100644 test/sharness/t0118-gateway-car/README.md delete mode 100644 test/sharness/t0118-gateway-car/carv1-basic.car delete mode 100644 test/sharness/t0118-gateway-car/carv1-basic.json delete mode 100644 test/sharness/t0118-gateway-car/deterministic.car delete mode 100644 test/sharness/t0118-gateway-car/test-dag.car diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 0f86d4da7..25ec19e0f 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -28,24 +28,21 @@ import ( func GatewayOption(paths ...string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - gwConfig, err := getGatewayConfig(n) + config, err := getGatewayConfig(n) if err != nil { return nil, err } - gwAPI, err := newGatewayBackend(n) + backend, err := newGatewayBackend(n) if err != nil { return nil, err } - gw := gateway.NewHandler(gwConfig, gwAPI) - gw = otelhttp.NewHandler(gw, "Gateway") - - // By default, our HTTP handler is the gateway handler. - handler := gw.ServeHTTP + handler := gateway.NewHandler(config, backend) + handler = otelhttp.NewHandler(handler, "Gateway") for _, p := range paths { - mux.HandleFunc(p+"/", handler) + mux.HandleFunc(p+"/", handler.ServeHTTP) } return mux, nil @@ -54,18 +51,18 @@ func GatewayOption(paths ...string) ServeOption { func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - gwConfig, err := getGatewayConfig(n) + config, err := getGatewayConfig(n) if err != nil { return nil, err } - gwAPI, err := newGatewayBackend(n) + backend, err := newGatewayBackend(n) if err != nil { return nil, err } childMux := http.NewServeMux() - mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP) + mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP) return childMux, nil } } @@ -111,11 +108,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { } } - gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) + backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) if err != nil { return nil, err } - return &offlineGatewayErrWrapper{gwimpl: gw}, nil + return &offlineGatewayErrWrapper{gwimpl: backend}, nil } type offlineGatewayErrWrapper struct { @@ -159,10 +156,10 @@ func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway return md, err } -func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - md, data, errCh, err := o.gwimpl.GetCAR(ctx, path) +func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { + md, data, err := o.gwimpl.GetCAR(ctx, path, params) err = offlineErrWrap(err) - return md, data, errCh, err + return md, data, err } func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool { @@ -191,12 +188,12 @@ var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil) var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"} -var subdomainGatewaySpec = &gateway.Specification{ +var subdomainGatewaySpec = &gateway.PublicGateway{ Paths: defaultPaths, UseSubdomains: true, } -var defaultKnownGateways = map[string]*gateway.Specification{ +var defaultKnownGateways = map[string]*gateway.PublicGateway{ "localhost": subdomainGatewaySpec, } @@ -218,7 +215,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) { Headers: headers, DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), NoDNSLink: cfg.Gateway.NoDNSLink, - PublicGateways: map[string]*gateway.Specification{}, + PublicGateways: map[string]*gateway.PublicGateway{}, } // Add default implicit known gateways, such as subdomain gateway on localhost. @@ -235,7 +232,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) { continue } - gwCfg.PublicGateways[hostname] = &gateway.Specification{ + gwCfg.PublicGateways[hostname] = &gateway.PublicGateway{ Paths: gw.Paths, NoDNSLink: gw.NoDNSLink, UseSubdomains: gw.UseSubdomains, diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index 55350caee..034201748 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -104,6 +104,10 @@ $ curl "https://subdomain-gw.example.net/ipfs/${cid}/" Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). +#### Gateway: support for CAR parameters of IPIP-402 + +The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402). + #### `ipfs dag stat` deduping statistics `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index ce14f58c3..97a6bedb5 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.9.0 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.5 github.com/multiformats/go-multiaddr v0.9.0 @@ -87,7 +87,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect - github.com/ipfs/go-unixfsnode v1.6.0 // indirect + github.com/ipfs/go-unixfsnode v1.7.1 // indirect github.com/ipld/edelweiss v0.2.0 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.20.0 // indirect @@ -139,6 +139,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openzipkin/zipkin-go v0.4.1 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -156,6 +157,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 3756be266..0a7693ed1 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -319,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM= -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -409,8 +409,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo= -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= @@ -421,6 +421,7 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= @@ -646,6 +647,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -781,6 +783,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= diff --git a/go.mod b/go.mod index 2d8266005..7e6386f1c 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.9.0 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 - github.com/ipfs/go-unixfsnode v1.6.0 + github.com/ipfs/go-unixfsnode v1.7.1 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c @@ -177,6 +177,7 @@ require ( github.com/onsi/ginkgo/v2 v2.9.2 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/openzipkin/zipkin-go v0.4.1 // indirect + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -197,6 +198,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect diff --git a/go.sum b/go.sum index 5ab048972..d1ca4000a 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM= -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -448,8 +448,8 @@ github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnz github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo= -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= @@ -715,6 +715,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -894,6 +895,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index e2ed3cada..2c667ab89 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.49.0 - github.com/ipfs/boxo v0.9.0 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 @@ -131,7 +131,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect - github.com/ipfs/go-unixfsnode v1.6.0 // indirect + github.com/ipfs/go-unixfsnode v1.7.1 // indirect github.com/ipfs/kubo v0.16.0 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 907caf115..199b20938 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -412,8 +412,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM= -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw= +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= @@ -465,8 +465,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo= -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/hang-fds v0.1.0 h1:deBiFlWHsVGzJ0ZMaqscEqRM1r2O1rFZ59UiQXb1Xko= github.com/ipfs/hang-fds v0.1.0/go.mod h1:29VLWOn3ftAgNNgXg/al7b11UzuQ+w7AwtCGcTaWkbM= diff --git a/test/sharness/t0118-gateway-car.sh b/test/sharness/t0118-gateway-car.sh deleted file mode 100755 index 7b7d998ee..000000000 --- a/test/sharness/t0118-gateway-car.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support" - -. lib/test-lib.sh - -test_init_ipfs -test_launch_ipfs_daemon_without_network - -# CAR stream is not deterministic, as blocks can arrive in random order, -# but if we have a small file that fits into a single block, and export its CID -# we will get a CAR that is a deterministic array of bytes. - -# Import test case -# See the static fixtures in ./t0118-gateway-car/ -test_expect_success "Add the dir test directory" ' - cp ../t0118-gateway-car/test-dag.car ./test-dag.car && - cp ../t0118-gateway-car/deterministic.car ./deterministic.car -' -ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./ -FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt - -# GET a reference DAG with dag-cbor+dag-pb+raw blocks as CAR - - # This test uses official CARv1 fixture from https://ipld.io/specs/transport/car/fixture/carv1-basic/ - test_expect_success "GET for application/vnd.ipld.car with dag-cbor root returns a CARv1 stream with full DAG" ' - ipfs dag import ../t0118-gateway-car/carv1-basic.car && - DAG_CBOR_CID=bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm && - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_CID" -o gateway-dag-cbor.car && - purge_blockstore && - ipfs dag import gateway-dag-cbor.car && - ipfs dag stat --offline $DAG_CBOR_CID - ' - -# GET unixfs file as CAR -# (by using a single file we ensure deterministic result that can be compared byte-for-byte) - - test_expect_success "GET with format=car param returns a CARv1 stream" ' - ipfs dag import test-dag.car && - curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car && - test_cmp deterministic.car gateway-param.car - ' - - test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" ' - ipfs dag import test-dag.car && - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car && - test_cmp deterministic.car gateway-header.car - ' - - # explicit version=1 - test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" ' - ipfs dag import test-dag.car && - curl -sX GET -H "Accept: application/vnd.ipld.car;version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car && - test_cmp deterministic.car gateway-header-v1.car - ' - - # explicit version=1 with whitepace - test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream (with whitespace)" ' - ipfs dag import test-dag.car && - curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car && - test_cmp deterministic.car gateway-header-v1.car - ' - - # explicit version=2 - test_expect_success "GET for application/vnd.ipld.raw version=2 returns HTTP 400 Bad Request error" ' - curl -svX GET -H "Accept: application/vnd.ipld.car;version=2" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" > curl_output 2>&1 && - cat curl_output && - grep "400 Bad Request" curl_output && - grep "unsupported CAR version" curl_output - ' - -# GET unixfs directory as a CAR with DAG and some selector - - # TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769 - test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" ' - ipfs dag import test-dag.car && - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car && - purge_blockstore && - ipfs dag import gateway-dir.car && - ipfs dag stat --offline $ROOT_DIR_CID - ' - -# Make sure expected HTTP headers are returned with the CAR bytes - - test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" ' - ipfs dag import test-dag.car && - curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output && - cat curl_output && - grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output - ' - - # CAR is streamed, gateway may not have the entire thing, unable to calculate total size - test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" ' - grep -qv "< Content-Length:" curl_output - ' - - test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" ' - grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output - ' - - test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" ' - grep "< X-Content-Type-Options: nosniff" curl_output - ' - - # CAR is streamed, gateway may not have the entire thing, unable to support range-requests - # Partial downloads and resumes should be handled using - # IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769 - test_expect_success "GET response for application/vnd.ipld.car includes Accept-Ranges header" ' - grep "< Accept-Ranges: none" curl_output - ' - - test_expect_success "GET for application/vnd.ipld.car with query filename includes Content-Disposition with custom filename" ' - curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?filename=foobar.car" >/dev/null 2>curl_output_filename && - cat curl_output_filename && - grep "< Content-Disposition: attachment\; filename=\"foobar.car\"" curl_output_filename - ' - -# Cache control HTTP headers - - test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" ' - grep "< Etag: W/\"${FILE_CID}.car\"" curl_output - ' - - # (basic checks, detailed behavior for some fields is tested in t0116-gateway-cache.sh) - test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" ' - grep "< X-Ipfs-Path" curl_output && - grep "< X-Ipfs-Roots" curl_output - ' - - test_expect_success "GET response for application/vnd.ipld.car includes same Cache-Control as a block or a file" ' - grep "< Cache-Control: public, max-age=29030400, immutable" curl_output - ' - -test_kill_ipfs_daemon - -test_done diff --git a/test/sharness/t0118-gateway-car/README.md b/test/sharness/t0118-gateway-car/README.md deleted file mode 100644 index 7b81e543b..000000000 --- a/test/sharness/t0118-gateway-car/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Dataset description/sources - -- carv1-basic.car - - raw CARv1 - - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.car - -- carv1-basic.json - - description of the contents and layout of the raw CAR, encoded in DAG-JSON - - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.json - -- test-dag.car + deterministic.car - - raw CARv1 - -generated with: - -```sh -# using ipfs version 0.18.1 -mkdir -p subdir && -echo "hello application/vnd.ipld.car" > subdir/ascii.txt && -ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) && -FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) && -ipfs dag export $ROOT_DIR_CID > test-dag.car && -ipfs dag export $FILE_CID > deterministic.car && - -echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./ -echo FILE_CID=${FILE_CID} # /\subdir/ascii.txt - -# ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./ -# FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt -``` diff --git a/test/sharness/t0118-gateway-car/carv1-basic.car b/test/sharness/t0118-gateway-car/carv1-basic.car deleted file mode 100644 index 48c67a3d8dc77ccff652289efb2b41edd4867d44..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 715 zcmYdZlv@T^XBM16^mDnYKs~9PRn;i&guKF^#9|+JO5wK+*E)0UP@kKZfZ(W zPG&(fBVpA-dR!`up+XAVeqSs7*{8Kv>B?Kpzb6a>{@wA2tebRKs!GK|)@4yipb$$^ zYGRQDi;zB-l8{2>%BFiZGufxw`Wo51ljYy|w%_%~d!}|Co}CXSx$O<+5@IV(P0r6t zk(kP;L5%6iK+phmEkSd2A+Bva6D`izRIm8 zgfZBugp5uCfurnv%3WfpBhW%hPLndXVc_T{3dI4x6g)Z xpWW$YsYS(^`FV`a#Hh|l&B@7ENGvGG$xKcx0cz7P%S+MAEXYaGOHM4}0stv?G@1Ya diff --git a/test/sharness/t0118-gateway-car/test-dag.car b/test/sharness/t0118-gateway-car/test-dag.car deleted file mode 100644 index e80fa4b0756cb7c9f5ebc44242424506fb7283cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 312 zcmcColv^5u?@7svMQ{B%`(=Of-ix^$+_m$U+s0a-OIe^X z=hdn7vecsD%=|pYC}LC#8E~n9)Saw+P!{lU+i#stX^z~mrGeJ6Qa_#6abJ&?`ByJ= zomD}It++HPC9_B(f{TfRF_tJT#z3P(g%nmX+uSVRD>}Pdpzx`&WNv8to_{t?F2!%M zR(JbssP@?{#FH}E7+o2uIXU?Xi3J5YnaPPIK>zBO<)!Fl7UZPp JB_|ef0RZo3fVKbt