coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-02-21 10:27:46 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

feat(gateway): _redirects file support (#8890)

Browse Source 
https://github.com/ipfs/kubo/pull/8890
https://github.com/ipfs/specs/pull/290
This commit is contained in:
Justin Johnson 2022-09-23 11:44:19 -05:00 committed by GitHub
parent 2549475d84
commit bcaacdd6c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 594 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/coreapi/name.go
View File

@ -37,6 +37,8 @@ func (e *ipnsEntry) Value() path.Path {
return e.value
}
type requestContextKey string
// Publish announces new IPNS name and returns the new IPNS entry.
func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (coreiface.IpnsEntry, error) {
ctx, span := tracing.Span(ctx, "CoreAPI.NameAPI", "Publish", trace.WithAttributes(attribute.String("path", p.String())))
@ -76,7 +78,7 @@ func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.Nam
if options.TTL != nil {
// nolint: staticcheck // non-backward compatible change
ctx = context.WithValue(ctx, "ipns-publish-ttl", *options.TTL)
ctx = context.WithValue(ctx, requestContextKey("ipns-publish-ttl"), *options.TTL)
}
eol := time.Now().Add(options.ValidTime)

138
core/corehttp/gateway_handler.go
View File

@ -13,7 +13,6 @@ import (
gopath "path"
"regexp"
"runtime/debug"
"strconv"
"strings"
"time"
@ -378,23 +377,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
return
}
// Resolve path to the final DAG node for the ETag
resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
switch err {
case nil:
case coreiface.ErrOffline:
webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
return
default:
// if Accept is text/html, see if ipfs-404.html is present
if i.servePretty404IfPresent(w, r, contentPath) {
logger.Debugw("serve pretty 404 if present")
return
}
webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusBadRequest)
return
}
// Detect when explicit Accept header or ?format parameter are present
responseFormat, formatParams, err := customResponseFormat(r)
if err != nil {
@ -402,6 +384,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
return
}
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
resolvedPath, contentPath, ok := i.handlePathResolution(w, r, responseFormat, contentPath, logger)
if !ok {
return
}
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))
// Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified
@ -450,36 +437,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
}
}
func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath)
if err != nil {
return false
}
dr, err := i.api.Unixfs().Get(r.Context(), resolved404Path)
if err != nil {
return false
}
defer dr.Close()
f, ok := dr.(files.File)
if !ok {
return false
}
size, err := f.Size()
if err != nil {
return false
}
log.Debugw("using pretty 404 file", "path", contentPath)
w.Header().Set("Content-Type", ctype)
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.WriteHeader(http.StatusNotFound)
_, err = io.CopyN(w, f, size)
return err == nil
}
func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
p, err := i.api.Unixfs().Add(r.Context(), files.NewReaderFile(r.Body))
if err != nil {
@ -920,48 +877,6 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
return "", nil, nil
}
func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) {
filename404, ctype, err := preferred404Filename(r.Header.Values("Accept"))
if err != nil {
return nil, "", err
}
pathComponents := strings.Split(contentPath.String(), "/")
for idx := len(pathComponents); idx >= 3; idx-- {
pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
parsed404Path := ipath.New("/" + pretty404)
if parsed404Path.IsValid() != nil {
break
}
resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path)
if err != nil {
continue
}
return resolvedPath, ctype, nil
}
return nil, "", fmt.Errorf("no pretty 404 in any parent folder")
}
func preferred404Filename(acceptHeaders []string) (string, string, error) {
// If we ever want to offer a 404 file for a different content type
// then this function will need to parse q weightings, but for now
// the presence of anything matching HTML is enough.
for _, acceptHeader := range acceptHeaders {
accepted := strings.Split(acceptHeader, ",")
for _, spec := range accepted {
contentType := strings.SplitN(spec, ";", 1)[0]
switch contentType {
case "*/*", "text/*", "text/html":
return "ipfs-404.html", "text/html", nil
}
}
}
return "", "", fmt.Errorf("there is no 404 file for the requested content types")
}
// returns unquoted path with all special characters revealed as \u codes
func debugStr(path string) string {
q := fmt.Sprintf("%+q", path)
@ -971,6 +886,49 @@ func debugStr(path string) string {
return q
}
// Resolve the provided contentPath including any special handling related to
// the requested responseFormat. Returned ok flag indicates if gateway handler
// should continue processing the request.
func (i *gatewayHandler) handlePathResolution(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, logger *zap.SugaredLogger) (resolvedPath ipath.Resolved, newContentPath ipath.Path, ok bool) {
// Attempt to resolve the provided path.
resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
switch err {
case nil:
return resolvedPath, contentPath, true
case coreiface.ErrOffline:
webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
return nil, nil, false
default:
// The path can't be resolved.
if isUnixfsResponseFormat(responseFormat) {
// If we have origin isolation (subdomain gw, DNSLink website),
// and response type is UnixFS (default for website hosting)
// check for presence of _redirects file and apply rules defined there.
// See: https://github.com/ipfs/specs/pull/290
if hasOriginIsolation(r) {
resolvedPath, newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, resolvedPath, contentPath, logger)
if hadMatchingRule {
logger.Debugw("applied a rule from _redirects file")
return resolvedPath, newContentPath, ok
}
}
// if Accept is text/html, see if ipfs-404.html is present
// This logic isn't documented and will likely be removed at some point.
// Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back
if i.serveLegacy404IfPresent(w, r, contentPath) {
logger.Debugw("served legacy 404")
return nil, nil, false
}
}
// Note: webError will replace http.StatusBadRequest with StatusNotFound if necessary
webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusBadRequest)
return nil, nil, false
}
}
// Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore.
// https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header
func (i *gatewayHandler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) (requestHandled bool) {

287
core/corehttp/gateway_handler_unixfs__redirects.go Normal file
View File

@ -0,0 +1,287 @@
package corehttp
import (
"fmt"
"io"
"net/http"
gopath "path"
"strconv"
"strings"
files "github.com/ipfs/go-ipfs-files"
redirects "github.com/ipfs/go-ipfs-redirects-file"
ipath "github.com/ipfs/interface-go-ipfs-core/path"
"go.uber.org/zap"
)
// Resolving a UnixFS path involves determining if the provided `path.Path` exists and returning the `path.Resolved`
// corresponding to that path. For UnixFS, path resolution is more involved.
//
// When a path under requested CID does not exist, Gateway will check if a `_redirects` file exists
// underneath the root CID of the path, and apply rules defined there.
// See sepcification introduced in: https://github.com/ipfs/specs/pull/290
//
// Scenario 1:
// If a path exists, we always return the `path.Resolved` corresponding to that path, regardless of the existence of a `_redirects` file.
//
// Scenario 2:
// If a path does not exist, usually we should return a `nil` resolution path and an error indicating that the path
// doesn't exist. However, a `_redirects` file may exist and contain a redirect rule that redirects that path to a different path.
// We need to evaluate the rule and perform the redirect if present.
//
// Scenario 3:
// Another possibility is that the path corresponds to a rewrite rule (i.e. a rule with a status of 200).
// In this case, we don't perform a redirect, but do need to return a `path.Resolved` and `path.Path` corresponding to
// the rewrite destination path.
//
// Note that for security reasons, redirect rules are only processed when the request has origin isolation.
// See https://github.com/ipfs/specs/pull/290 for more information.
func (i *gatewayHandler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) (newResolvedPath ipath.Resolved, newContentPath ipath.Path, continueProcessing bool, hadMatchingRule bool) {
redirectsFile := i.getRedirectsFile(r, contentPath, logger)
if redirectsFile != nil {
redirectRules, err := i.getRedirectRules(r, redirectsFile)
if err != nil {
internalWebError(w, err)
return nil, nil, false, true
}
redirected, newPath, err := i.handleRedirectsFileRules(w, r, contentPath, redirectRules)
if err != nil {
err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsFile.String(), err)
internalWebError(w, err)
return nil, nil, false, true
}
if redirected {
return nil, nil, false, true
}
// 200 is treated as a rewrite, so update the path and continue
if newPath != "" {
// Reassign contentPath and resolvedPath since the URL was rewritten
contentPath = ipath.New(newPath)
resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath)
if err != nil {
internalWebError(w, err)
return nil, nil, false, true
}
return resolvedPath, contentPath, true, true
}
}
// No matching rule, paths remain the same, continue regular processing
return resolvedPath, contentPath, true, false
}
func (i *gatewayHandler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, redirectRules []redirects.Rule) (redirected bool, newContentPath string, err error) {
// Attempt to match a rule to the URL path, and perform the corresponding redirect or rewrite
pathParts := strings.Split(contentPath.String(), "/")
if len(pathParts) > 3 {
// All paths should start with /ipfs/cid/, so get the path after that
urlPath := "/" + strings.Join(pathParts[3:], "/")
rootPath := strings.Join(pathParts[:3], "/")
// Trim off the trailing /
urlPath = strings.TrimSuffix(urlPath, "/")
for _, rule := range redirectRules {
// Error right away if the rule is invalid
if !rule.MatchAndExpandPlaceholders(urlPath) {
continue
}
// We have a match!
// Rewrite
if rule.Status == 200 {
// Prepend the rootPath
toPath := rootPath + rule.To
return false, toPath, nil
}
// Or 4xx
if rule.Status == 404 || rule.Status == 410 || rule.Status == 451 {
toPath := rootPath + rule.To
content4xxPath := ipath.New(toPath)
err := i.serve4xx(w, r, content4xxPath, rule.Status)
return true, toPath, err
}
// Or redirect
if rule.Status >= 301 && rule.Status <= 308 {
http.Redirect(w, r, rule.To, rule.Status)
return true, "", nil
}
}
}
// No redirects matched
return false, "", nil
}
func (i *gatewayHandler) getRedirectRules(r *http.Request, redirectsFilePath ipath.Resolved) ([]redirects.Rule, error) {
// Convert the path into a file node
node, err := i.api.Unixfs().Get(r.Context(), redirectsFilePath)
if err != nil {
return nil, fmt.Errorf("could not get _redirects: %w", err)
}
defer node.Close()
// Convert the node into a file
f, ok := node.(files.File)
if !ok {
return nil, fmt.Errorf("could not parse _redirects: %w", err)
}
// Parse redirect rules from file
redirectRules, err := redirects.Parse(f)
if err != nil {
return nil, fmt.Errorf("could not parse _redirects: %w", err)
}
return redirectRules, nil
}
// Returns a resolved path to the _redirects file located in the root CID path of the requested path
func (i *gatewayHandler) getRedirectsFile(r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) ipath.Resolved {
// contentPath is the full ipfs path to the requested resource,
// regardless of whether path or subdomain resolution is used.
rootPath := getRootPath(contentPath)
// Check for _redirects file.
// Any path resolution failures are ignored and we just assume there's no _redirects file.
// Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail.
path := ipath.Join(rootPath, "_redirects")
resolvedPath, err := i.api.ResolvePath(r.Context(), path)
if err != nil {
return nil
}
return resolvedPath
}
// Returns the root CID Path for the given path
func getRootPath(path ipath.Path) ipath.Path {
parts := strings.Split(path.String(), "/")
return ipath.New(gopath.Join("/", path.Namespace(), parts[2]))
}
func (i *gatewayHandler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPath ipath.Path, status int) error {
resolved4xxPath, err := i.api.ResolvePath(r.Context(), content4xxPath)
if err != nil {
return err
}
node, err := i.api.Unixfs().Get(r.Context(), resolved4xxPath)
if err != nil {
return err
}
defer node.Close()
f, ok := node.(files.File)
if !ok {
return fmt.Errorf("could not convert node for %d page to file", status)
}
size, err := f.Size()
if err != nil {
return fmt.Errorf("could not get size of %d page", status)
}
log.Debugf("using _redirects: custom %d file at %q", status, content4xxPath)
w.Header().Set("Content-Type", "text/html")
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
addCacheControlHeaders(w, r, content4xxPath, resolved4xxPath.Cid())
w.WriteHeader(status)
_, err = io.CopyN(w, f, size)
return err
}
func hasOriginIsolation(r *http.Request) bool {
_, gw := r.Context().Value(requestContextKey("gw-hostname")).(string)
_, dnslink := r.Context().Value("dnslink-hostname").(string)
if gw || dnslink {
return true
}
return false
}
func isUnixfsResponseFormat(responseFormat string) bool {
// The implicit response format is UnixFS
return responseFormat == ""
}
// Deprecated: legacy ipfs-404.html files are superseded by _redirects file
// This is provided only for backward-compatibility, until websites migrate
// to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290)
func (i *gatewayHandler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath)
if err != nil {
return false
}
dr, err := i.api.Unixfs().Get(r.Context(), resolved404Path)
if err != nil {
return false
}
defer dr.Close()
f, ok := dr.(files.File)
if !ok {
return false
}
size, err := f.Size()
if err != nil {
return false
}
log.Debugw("using pretty 404 file", "path", contentPath)
w.Header().Set("Content-Type", ctype)
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.WriteHeader(http.StatusNotFound)
_, err = io.CopyN(w, f, size)
return err == nil
}
func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) {
filename404, ctype, err := preferred404Filename(r.Header.Values("Accept"))
if err != nil {
return nil, "", err
}
pathComponents := strings.Split(contentPath.String(), "/")
for idx := len(pathComponents); idx >= 3; idx-- {
pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
parsed404Path := ipath.New("/" + pretty404)
if parsed404Path.IsValid() != nil {
break
}
resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path)
if err != nil {
continue
}
return resolvedPath, ctype, nil
}
return nil, "", fmt.Errorf("no pretty 404 in any parent folder")
}
func preferred404Filename(acceptHeaders []string) (string, string, error) {
// If we ever want to offer a 404 file for a different content type
// then this function will need to parse q weightings, but for now
// the presence of anything matching HTML is enough.
for _, acceptHeader := range acceptHeaders {
accepted := strings.Split(acceptHeader, ",")
for _, spec := range accepted {
contentType := strings.SplitN(spec, ";", 1)[0]
switch contentType {
case "*/*", "text/*", "text/html":
return "ipfs-404.html", "text/html", nil
}
}
}
return "", "", fmt.Errorf("there is no 404 file for the requested content types")
}

2
core/corehttp/gateway_handler_unixfs_dir.go
View File

@ -185,7 +185,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
var gwURL string
// Get gateway hostname and build gateway URL.
if h, ok := r.Context().Value("gw-hostname").(string); ok {
if h, ok := r.Context().Value(requestContextKey("gw-hostname")).(string); ok {
gwURL = "//" + h
} else {
gwURL = ""

7
core/corehttp/hostname.go
View File

@ -221,7 +221,8 @@ func HostnameOption() ServeOption {
if !cfg.Gateway.NoDNSLink && isDNSLinkName(r.Context(), coreAPI, host) {
// rewrite path and handle as DNSLink
r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path
childMux.ServeHTTP(w, withHostnameContext(r, host))
ctx := context.WithValue(r.Context(), requestContextKey("dnslink-hostname"), host)
childMux.ServeHTTP(w, withHostnameContext(r.WithContext(ctx), host))
return
}
@ -242,6 +243,8 @@ type wildcardHost struct {
spec *config.GatewaySpec
}
type requestContextKey string
// Extends request context to include hostname of a canonical gateway root
// (subdomain root or dnslink fqdn)
func withHostnameContext(r *http.Request, hostname string) *http.Request {
@ -250,7 +253,7 @@ func withHostnameContext(r *http.Request, hostname string) *http.Request {
// Host header, subdomain gateways have more comples rules (knownSubdomainDetails)
// More: https://github.com/ipfs/dir-index-html/issues/42
// nolint: staticcheck // non-backward compatible change
ctx := context.WithValue(r.Context(), "gw-hostname", hostname)
ctx := context.WithValue(r.Context(), requestContextKey("gw-hostname"), hostname)
return r.WithContext(ctx)
}

4
docs/examples/kubo-as-a-library/go.sum
View File

@ -571,6 +571,7 @@ github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY
github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
github.com/ipfs/go-ipfs-provider v0.7.1 h1:eKToBUAb6ZY8iiA6AYVxzW4G1ep67XUaaEBUIYpxhfw=
github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8=
github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY=
@ -1535,9 +1536,11 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@ -2179,6 +2182,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

2
go.mod
View File

@ -117,6 +117,7 @@ require (
require (
github.com/benbjohnson/clock v1.3.0
github.com/ipfs/go-delegated-routing v0.6.0
github.com/ipfs/go-ipfs-redirects-file v0.1.1
github.com/ipfs/go-log/v2 v2.5.1
)
@ -225,6 +226,7 @@ require (
github.com/tidwall/gjson v1.14.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect

5
go.sum
View File

@ -565,6 +565,8 @@ github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY
github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
github.com/ipfs/go-ipfs-provider v0.7.1 h1:eKToBUAb6ZY8iiA6AYVxzW4G1ep67XUaaEBUIYpxhfw=
github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8=
github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8=
github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY=
@ -1511,9 +1513,12 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ=
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

BIN
test/sharness/t0109-gateway-web-_redirects-data/redirects.car Normal file
View File

Binary file not shown.

239
test/sharness/t0109-gateway-web-_redirects.sh Executable file
View File

@ -0,0 +1,239 @@
#!/usr/bin/env bash
test_description="Test HTTP Gateway _redirects support"
. lib/test-lib.sh
test_init_ipfs
test_launch_ipfs_daemon
## ============================================================================
## Test _redirects file support
## ============================================================================
# Import test case
# Run `ipfs cat /ipfs/$REDIRECTS_DIR_CID/_redirects` to see sample _redirects file
test_expect_success "Add the _redirects file test directory" '
ipfs dag import ../t0109-gateway-web-_redirects-data/redirects.car
'
CAR_ROOT_CID=QmQyqMY5vUBSbSxyitJqthgwZunCQjDVtNd8ggVCxzuPQ4
REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3)
REDIRECTS_DIR_HOSTNAME="${REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/redirect-one" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /one.html" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/301-redirect-one" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /one.html" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/302-redirect-two" > response &&
test_should_contain "302 Found" response &&
test_should_contain "Location: /two.html" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/200-index" > response &&
test_should_contain "my index" response &&
test_should_contain "200 OK" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/posts/2022/01/01/hello-world" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /articles/2022/01/01/hello-world" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/splat/one.html" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /redirected-splat/one.html" response
'
# ensure custom 4xx works and has the same cache headers as regular /ipfs/ path
CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/404.html | cut -d "/" -f3)
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry" > response &&
test_should_contain "404 Not Found" response &&
test_should_contain "Cache-Control: public, max-age=29030400, immutable" response &&
test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response &&
test_should_contain "my 404" response
'
CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3)
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry" > response &&
test_should_contain "410 Gone" response &&
test_should_contain "Cache-Control: public, max-age=29030400, immutable" response &&
test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response &&
test_should_contain "my 410" response
'
CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3)
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry" > response &&
test_should_contain "451 Unavailable For Legal Reasons" response &&
test_should_contain "Cache-Control: public, max-age=29030400, immutable" response &&
test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response &&
test_should_contain "my 451" response
'
test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" '
curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/catch-all" > response &&
test_should_contain "200 OK" response &&
test_should_contain "my index" response
'
# This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)
test_expect_success "request for http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one returns generic 404 (no custom 404 from _redirects since no origin isolation)" '
curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one" > response &&
test_should_contain "404 Not Found" response &&
test_should_not_contain "my 404" response
'
# With CRLF line terminator
NEWLINE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/newlines | cut -d "/" -f3)
NEWLINE_REDIRECTS_DIR_HOSTNAME="${NEWLINE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
test_expect_success "newline: _redirects has CRLF line terminators" '
ipfs cat /ipfs/$NEWLINE_REDIRECTS_DIR_CID/_redirects | file - > response &&
test_should_contain "with CRLF line terminators" response
'
test_expect_success "newline: request for $NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" '
curl -sD - --resolve $NEWLINE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /one.html" response
'
# Good codes
GOOD_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/good-codes | cut -d "/" -f3)
GOOD_REDIRECTS_DIR_HOSTNAME="${GOOD_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
test_expect_success "good codes: request for $GOOD_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" '
curl -sD - --resolve $GOOD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$GOOD_REDIRECTS_DIR_HOSTNAME/a301" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /b301" response
'
# Bad codes
BAD_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/bad-codes | cut -d "/" -f3)
BAD_REDIRECTS_DIR_HOSTNAME="${BAD_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
# if accessing a path that doesn't exist, read _redirects and fail parsing, and return error
test_expect_success "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/not-found returns error about bad code" '
curl -sD - --resolve $BAD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$BAD_REDIRECTS_DIR_HOSTNAME/not-found" > response &&
test_should_contain "500" response &&
test_should_contain "status code 999 is not supported" response
'
# if accessing a path that does exist, don't read _redirects and therefore don't fail parsing
test_expect_success "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/found.html doesn't return error about bad code" '
curl -sD - --resolve $BAD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$BAD_REDIRECTS_DIR_HOSTNAME/found.html" > response &&
test_should_contain "200" response &&
test_should_contain "my found" response &&
test_should_not_contain "unsupported redirect status" response
'
# Invalid file, containing "hello"
INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/invalid | cut -d "/" -f3)
INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
# if accessing a path that doesn't exist, read _redirects and fail parsing, and return error
test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" '
curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response &&
test_should_contain "500" response &&
test_should_contain "could not parse _redirects:" response
'
# Invalid file, containing forced redirect
INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/forced | cut -d "/" -f3)
INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
# if accessing a path that doesn't exist, read _redirects and fail parsing, and return error
test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" '
curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response &&
test_should_contain "500" response &&
test_should_contain "could not parse _redirects:" response &&
test_should_contain "forced redirects (or \"shadowing\") are not supported" response
'
# if accessing a path that doesn't exist and _redirects file is too large, return error
TOO_LARGE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/too-large | cut -d "/" -f3)
TOO_LARGE_REDIRECTS_DIR_HOSTNAME="${TOO_LARGE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT"
test_expect_success "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file" '
curl -sD - --resolve $TOO_LARGE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found" > response &&
test_should_contain "500" response &&
test_should_contain "could not parse _redirects:" response &&
test_should_contain "redirects file size cannot exceed" response
'
test_kill_ipfs_daemon
# disable wildcard DNSLink gateway
# and enable it on specific DNSLink hostname
ipfs config --json Gateway.NoDNSLink true && \
ipfs config --json Gateway.PublicGateways '{
"dnslink-enabled-on-fqdn.example.org": {
"NoDNSLink": false,
"UseSubdomains": false,
"Paths": ["/ipfs"]
},
"dnslink-disabled-on-fqdn.example.com": {
"NoDNSLink": true,
"UseSubdomains": false,
"Paths": []
}
}' || exit 1
# DNSLink test requires a daemon in online mode with precached /ipns/ mapping
# REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3)
DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org"
NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com"
export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$REDIRECTS_DIR_CID"
# restart daemon to apply config changes
test_launch_ipfs_daemon
# make sure test setup is valid (fail if CoreAPI is unable to resolve)
test_expect_success "spoofed DNSLink record resolves in cli" "
ipfs resolve /ipns/$DNSLINK_FQDN > result &&
test_should_contain \"$REDIRECTS_DIR_CID\" result &&
ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result &&
test_should_contain \"index.html\" result
"
test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" '
curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
test_should_contain "301 Moved Permanently" response &&
test_should_contain "Location: /one.html" response
'
# ensure custom 404 works and has the same cache headers as regular /ipns/ paths
test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" '
curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response &&
test_should_contain "404 Not Found" response &&
test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response &&
test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response &&
test_should_not_contain "immutable" response &&
test_should_contain "Date: " response &&
test_should_contain "my 404" response
'
test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" '
curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
test_should_not_contain "one.html" response &&
test_should_not_contain "301 Moved Permanently" response &&
test_should_not_contain "Location:" response
'
test_kill_ipfs_daemon
test_done