feat(gateway): Gateway.FastDirIndexThreshold (#8853)

* fix(core/gateway): option to limit directory size listing * feat(gw): HTMLDirListingLimit This is alternative take on the way we limit the HTML listing output. Instead of a hard cut-off, we list up to HTMLDirListingLimit. When a directory has more items than HTMLDirListingLimit we show additional header and footer informing user that only $HTMLDirListingLimit items are listed. This is a better UX. * fix: 0 disables Gateway.HTMLDirListingLimit * refactor: Gateway.FastDirIndexThreshold see explainer in docs/config.md * refactor: prealoc slices * docs: Gateway.FastDirIndexThreshold * refactor: core/corehttp/gateway_handler.go https://github.com/ipfs/go-ipfs/pull/8853#discussion_r851437088 * docs: apply suggestions from code review Co-authored-by: Alan Shaw <alan.shaw@protocol.ai> Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>
2026-02-21 10:27:46 +08:00 · 2022-04-28 14:36:57 -03:00 · 2022-04-28 14:36:57 -03:00 · 25cc85fa93
commit 25cc85fa93
parent 4f7d4bcc05
12 changed files with 129 additions and 55 deletions
--- a/assets/dir-index-html/dir-index.html
+++ b/assets/dir-index-html/dir-index.html
@ -31,7 +31,7 @@
      <div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
      <div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
      <div>
-        <a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
+        <a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
        </a>
      </div>
--- a/assets/dir-index-html/src/dir-index.html
+++ b/assets/dir-index-html/src/dir-index.html
@ -30,7 +30,7 @@
      <div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
      <div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
      <div>
-        <a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
+        <a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
        </a>
      </div>
--- a/assets/dir-index-html/test/main.go
+++ b/assets/dir-index-html/test/main.go
@ -12,14 +12,15 @@ const templateFile = "../dir-index.html"

 // Copied from go-ipfs/core/corehttp/gateway_indexPage.go
 type listingTemplateData struct {
-	GatewayURL  string
-	DNSLink     bool
-	Listing     []directoryItem
-	Size        string
-	Path        string
-	Breadcrumbs []breadcrumb
-	BackLink    string
-	Hash        string
+	GatewayURL            string
+	DNSLink               bool
+	Listing               []directoryItem
+	Size                  string
+	Path                  string
+	Breadcrumbs           []breadcrumb
+	BackLink              string
+	Hash                  string
+	FastDirIndexThreshold int
 }

 type directoryItem struct {
--- a/config/gateway.go
+++ b/config/gateway.go
@ -53,7 +53,15 @@ type Gateway struct {
 	//  }
 	PathPrefixes []string

-	// FIXME: Not yet implemented
+	// FastDirIndexThreshold is the maximum number of items in a directory
+	// before the Gateway switches to a shallow, faster listing which only
+	// requires the root node. This allows for listing big directories fast,
+	// without the linear slowdown caused by reading size metadata from child
+	// nodes.
+	// Setting to 0 will enable fast listings for all directories.
+	FastDirIndexThreshold *OptionalInteger `json:",omitempty"`
+
+	// FIXME: Not yet implemented: https://github.com/ipfs/go-ipfs/issues/8059
 	APICommands []string

 	// NoFetch configures the gateway to _not_ fetch blocks in response to
--- a/core/coreapi/unixfs.go
+++ b/core/coreapi/unixfs.go
@ -302,7 +302,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
 }

 func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) {
-	out := make(chan coreiface.DirEntry)
+	out := make(chan coreiface.DirEntry, uio.DefaultShardWidth)

 	go func() {
 		defer close(out)
--- a/core/corehttp/gateway.go
+++ b/core/corehttp/gateway.go
@ -16,9 +16,10 @@ import (
 )

 type GatewayConfig struct {
-	Headers      map[string][]string
-	Writable     bool
-	PathPrefixes []string
+	Headers               map[string][]string
+	Writable              bool
+	PathPrefixes          []string
+	FastDirIndexThreshold int
 }

 // A helper function to clean up a set of headers:
@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			}, headers[ACEHeadersName]...))

 		var gateway http.Handler = newGatewayHandler(GatewayConfig{
-			Headers:      headers,
-			Writable:     writable,
-			PathPrefixes: cfg.Gateway.PathPrefixes,
+			Headers:               headers,
+			Writable:              writable,
+			PathPrefixes:          cfg.Gateway.PathPrefixes,
+			FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
 		}, api)

 		gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
--- a/core/corehttp/gateway_handler_unixfs.go
+++ b/core/corehttp/gateway_handler_unixfs.go
@ -18,6 +18,7 @@ import (
 func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
 	ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
 	defer span.End()
+
 	// Handling UnixFS
 	dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
 	if err != nil {
@ -39,6 +40,7 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
 		internalWebError(w, fmt.Errorf("unsupported UnixFS type"))
 		return
 	}
+
 	logger.Debugw("serving unixfs directory", "path", contentPath)
 	i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
 }
--- a/core/corehttp/gateway_handler_unixfs_dir.go
+++ b/core/corehttp/gateway_handler_unixfs_dir.go
@ -15,6 +15,7 @@ import (
 	"github.com/ipfs/go-ipfs/tracing"
 	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@ -102,36 +103,46 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 		return
 	}

-	// storage for directory listing
-	var dirListing []directoryItem
-	dirit := dir.Entries()
-	for dirit.Next() {
-		size := "?"
-		if s, err := dirit.Node().Size(); err == nil {
-			// Size may not be defined/supported. Continue anyways.
-			size = humanize.Bytes(uint64(s))
-		}
+	// Optimization 1:
+	// List children without fetching their root blocks (fast, but no size info)
+	results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
+	if err != nil {
+		internalWebError(w, err)
+		return
+	}

-		resolved, err := i.api.ResolvePath(ctx, ipath.Join(resolvedPath, dirit.Name()))
-		if err != nil {
+	// storage for directory listing
+	dirListing := make([]directoryItem, 0, len(results))
+
+	for link := range results {
+		if link.Err != nil {
 			internalWebError(w, err)
 			return
 		}
-		hash := resolved.Cid().String()
-
-		// See comment above where originalUrlPath is declared.
+		hash := link.Cid.String()
 		di := directoryItem{
-			Size:      size,
-			Name:      dirit.Name(),
-			Path:      gopath.Join(originalUrlPath, dirit.Name()),
+			Size:      "", // no size because we did not fetch child nodes
+			Name:      link.Name,
+			Path:      gopath.Join(originalUrlPath, link.Name),
 			Hash:      hash,
 			ShortHash: shortHash(hash),
 		}
 		dirListing = append(dirListing, di)
 	}
-	if dirit.Err() != nil {
-		internalWebError(w, dirit.Err())
-		return
+
+	// Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
+	if len(dirListing) < i.config.FastDirIndexThreshold {
+		dirit := dir.Entries()
+		linkNo := 0
+		for dirit.Next() {
+			size := "?"
+			if s, err := dirit.Node().Size(); err == nil {
+				// Size may not be defined/supported. Continue anyways.
+				size = humanize.Bytes(uint64(s))
+			}
+			dirListing[linkNo].Size = size
+			linkNo++
+		}
 	}

 	// construct the correct back link
@ -180,14 +191,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit

 	// See comment above where originalUrlPath is declared.
 	tplData := listingTemplateData{
-		GatewayURL:  gwURL,
-		DNSLink:     dnslink,
-		Listing:     dirListing,
-		Size:        size,
-		Path:        contentPath.String(),
-		Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
-		BackLink:    backLink,
-		Hash:        hash,
+		GatewayURL:            gwURL,
+		DNSLink:               dnslink,
+		Listing:               dirListing,
+		Size:                  size,
+		Path:                  contentPath.String(),
+		Breadcrumbs:           breadcrumbs(contentPath.String(), dnslink),
+		BackLink:              backLink,
+		Hash:                  hash,
+		FastDirIndexThreshold: i.config.FastDirIndexThreshold,
 	}

 	logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
--- a/core/corehttp/gateway_indexPage.go
+++ b/core/corehttp/gateway_indexPage.go
@ -12,14 +12,15 @@ import (

 // structs for directory listing
 type listingTemplateData struct {
-	GatewayURL  string
-	DNSLink     bool
-	Listing     []directoryItem
-	Size        string
-	Path        string
-	Breadcrumbs []breadcrumb
-	BackLink    string
-	Hash        string
+	GatewayURL            string
+	DNSLink               bool
+	Listing               []directoryItem
+	Size                  string
+	Path                  string
+	Breadcrumbs           []breadcrumb
+	BackLink              string
+	Hash                  string
+	FastDirIndexThreshold int
 }

 type directoryItem struct {
--- a/docs/config.md
+++ b/docs/config.md
@ -51,6 +51,7 @@ config file at runtime.
    - [`Gateway.NoDNSLink`](#gatewaynodnslink)
    - [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
    - [`Gateway.RootRedirect`](#gatewayrootredirect)
+    - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
    - [`Gateway.Writable`](#gatewaywritable)
    - [`Gateway.PathPrefixes`](#gatewaypathprefixes)
    - [`Gateway.PublicGateways`](#gatewaypublicgateways)
@ -646,6 +647,20 @@ Default: `""`

 Type: `string` (url)

+### `Gateway.FastDirIndexThreshold`
+
+The maximum number of items in a directory before the Gateway switches
+to a shallow, faster listing which only requires the root node.
+
+This allows for fast listings of big directories, without the linear slowdown caused
+by reading size metadata from child nodes.
+
+Setting to 0 will enable fast listings for all directories.
+
+Default: `100`
+
+Type: `optionalInteger`
+
 ### `Gateway.Writable`

 A boolean to configure whether the gateway is writeable or not.
--- a/test/sharness/lib/test-lib.sh
+++ b/test/sharness/lib/test-lib.sh
@ -388,6 +388,15 @@ test_should_contain() {
  fi
 }

+test_should_not_contain() {
+  test "$#" = 2 || error "bug in the test script: not 2 parameters to test_should_not_contain"
+  if grep -q "$1" "$2"
+  then
+    echo "'$2' contains undesired value '$1'"
+    return 1
+  fi
+}
+
 test_str_contains() {
  find=$1
  shift
--- a/test/sharness/t0115-gateway-dir-listing.sh
+++ b/test/sharness/t0115-gateway-dir-listing.sh
@ -28,7 +28,9 @@ test_expect_success "Add the test directory" '
  echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
  echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
  DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
-  FILE_CID=$(ipfs files stat /ipfs/$DIR_CID/ą/ę/file-źł.txt | head -1)
+  FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
+  FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
+  echo "$FILE_CID / $FILE_SIZE"
 '

 ## ============================================================================
@ -135,6 +137,28 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.io
  test_should_contain "<a class=\"ipfs-hash\" translate=\"no\" href=\"https://cid.ipfs.io/#$FILE_CID\" target=\"_blank\" rel=\"noreferrer noopener\">" list_response
 '

+## ============================================================================
+## Test dir listing of a big directory
+## ============================================================================
+
+test_expect_success "dir listing should resolve child sizes if under Gateway.FastDirIndexThreshold" '
+  curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
+  test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
+  test_should_contain ">${FILE_SIZE} B</td>" list_response
+'
+
+# force fast dir index for all responses
+ipfs config --json Gateway.FastDirIndexThreshold 0
+# restart daemon to apply config changes
+test_kill_ipfs_daemon
+test_launch_ipfs_daemon
+
+test_expect_success "dir listing should not resolve child sizes beyond Gateway.FastDirIndexThreshold" '
+  curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
+  test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
+  test_should_not_contain ">${FILE_SIZE} B</td>" list_response
+'
+
 ## ============================================================================
 ## End of tests, cleanup
 ## ============================================================================