Merge pull request #9936 from ipfs/release-v0.21.0

.github/pull_request_template.md

@@ -0,0 +1,4 @@
+<!--
+PR Creation Checklist
+- [ ] Update Changelog
+-->

.github/workflows/build.yml

@@ -3,17 +3,23 @@ name: Interop
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
 
 env:
-  GO_VERSION: 1.19.1
+  GO_VERSION: 1.19.x
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   interop-prep:
     if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
@@ -44,7 +50,7 @@ jobs:
           path: cmd/ipfs/ipfs
   interop:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
@@ -75,7 +81,7 @@ jobs:
           npm install ipfs-interop@^10.0.1
         working-directory: interop
       # Run the interop tests while ignoring the js-js interop test cases
-      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))'
+      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' --parallel
         env:
           LIBP2P_TCP_REUSEPORT: false
           LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
@@ -121,42 +127,9 @@ jobs:
         working-directory: go-ipfs-api
       - run: cmd/ipfs/ipfs shutdown
         if: always()
-  go-ipfs-http-client:
-    needs: [interop-prep]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      TEST_DOCKER: 0
-      TEST_FUSE: 0
-      TEST_VERBOSE: 1
-      TRAVIS: 1
-      GIT_PAGER: cat
-      IPFS_CHECK_RCMGR_DEFAULTS: 1
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: kubo
-          path: cmd/ipfs
-      - run: chmod +x cmd/ipfs/ipfs
-      - uses: actions/checkout@v3
-        with:
-          repository: ipfs/go-ipfs-http-client
-          path: go-ipfs-http-client
-      - uses: protocol/cache-go-action@v1
-        with:
-          name: ${{ github.job }}
-      - run: echo '${{ github.workspace }}/cmd/ipfs' >> $GITHUB_PATH
-      - run: go test -count=1 -v ./...
-        working-directory: go-ipfs-http-client
   ipfs-webui:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       NO_SANDBOX: true
@@ -191,14 +164,26 @@ jobs:
           key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-${{ github.job }}-
-      - run: |
-          npm ci --prefer-offline --no-audit --progress=false
-          npx playwright install
+      - env:
+          NPM_CACHE_DIR: ${{ steps.npm-cache-dir.outputs.dir }}
+        run: |
+          npm ci --prefer-offline --no-audit --progress=false --cache "$NPM_CACHE_DIR"
+          npx playwright install --with-deps
         working-directory: ipfs-webui
-      - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
-        run: npm test
+      - id: ref
+        run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT
         working-directory: ipfs-webui
-      - name: Test ipfs-webui@main E2E against the locally built Kubo binary
+      - id: state
+        env:
+          GITHUB_REPOSITORY: ipfs/ipfs-webui
+          GITHUB_REF: ${{ steps.ref.outputs.ref }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          echo "state=$(curl -L -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_REF/status" --jq '.state')" | tee -a $GITHUB_OUTPUT
+      - name: Build ipfs-webui@main (state=${{ steps.state.outputs.state }})
+        run: npm run test:build
+        working-directory: ipfs-webui
+      - name: Test ipfs-webui@main (state=${{ steps.state.outputs.state }}) E2E against the locally built Kubo binary
         run: npm run test:e2e
         env:
           IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs

.github/workflows/codeql-analysis.yml

@@ -8,6 +8,8 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ master ]
+    paths-ignore:
+      - '**/*.md'
   schedule:
     - cron: '30 12 * * 2'
 

.github/workflows/docker-build.yml

@@ -3,6 +3,8 @@ name: Docker Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -25,6 +27,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - run: docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .

.github/workflows/gateway-conformance.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - master
   pull_request:
+    paths-ignore:
+      - '**/*.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
@@ -21,7 +23,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.0
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.1
         with:
           output: fixtures
 
@@ -38,8 +40,8 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Start the kubo-gateway
-      - name: Start kubo-gateway
+      # 3. Init the kubo-gateway
+      - name: Init kubo-gateway
         env:
           GATEWAY_PUBLIC_GATEWAYS: |
             {
@@ -56,16 +58,36 @@ jobs:
         run: |
           ./ipfs init
           ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
-          ./ipfs daemon --offline &
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
       - name: Import fixtures
-        run: find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
 
-      # 5. Run the gateway-conformance tests
+          # Import ipns records
+          records=$(find ./fixtures -name '*.ipns-record')
+          for record in $records
+          do
+              key=$(basename -s .ipns-record "$record" | cut -d'_' -f1)
+              kubo-gateway/cmd/ipfs/ipfs routing put --allow-offline "/ipns/$key" "$record"
+          done
+
+          # Import dnslink records
+          # the IPFS_NS_MAP env will be used by the daemon
+          export IPFS_NS_MAP=$(cat ./fixtures/dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")')
+          echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV
+
+      # 5. Start the kubo-gateway
+      - name: Start kubo-gateway
+        run: |
+          ./ipfs daemon --offline &
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 6. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.0
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.1
         with:
           gateway-url: http://127.0.0.1:8080
           json: output.json
@@ -74,7 +96,7 @@ jobs:
           markdown: output.md
           args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
 
-      # 6. Upload the results
+      # 7. Upload the results
       - name: Upload MD summary
         if: failure() || success()
         run: cat output.md >> $GITHUB_STEP_SUMMARY

.github/workflows/gobuild.yml

@@ -3,6 +3,8 @@ name: Go Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-build-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-build:
-    needs: [go-build-runner]
-    runs-on: ${{ fromJSON(needs.go-build-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0
@@ -31,7 +30,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - uses: protocol/cache-go-action@v1
         with:

.github/workflows/golang-analysis.yml

@@ -3,6 +3,8 @@ name: Go Check
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'

.github/workflows/golint.yml

@@ -3,6 +3,8 @@ name: Go Lint
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -29,7 +31,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - uses: protocol/cache-go-action@v1
         with:

.github/workflows/gotest.yml

@@ -3,6 +3,8 @@ name: Go Test
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-test-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-test:
-    needs: [go-test-runner]
-    runs-on: ${{ fromJSON(needs.go-test-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0
@@ -33,7 +32,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - name: Check out Kubo
         uses: actions/checkout@v3
       - name: Restore Go cache
@@ -41,8 +40,11 @@ jobs:
         with:
           name: ${{ github.job }}
       - name: 👉️ If this step failed, go to «Summary» (top left) → inspect the «Failures/Errors» table
+        env:
+          # increasing parallelism beyond 2 doesn't speed up the tests much
+          PARALLEL: 2
         run: |
-          make -j 2 test/unit/gotest.junit.xml &&
+          make -j "$PARALLEL" test/unit/gotest.junit.xml &&
             [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0

.github/workflows/runner.yml

@@ -1,34 +0,0 @@
-name: Runner
-
-on:
-  workflow_call:
-    outputs:
-      config:
-        description: "The runner's configuration"
-        value: ${{ jobs.choose.outputs.config }}
-
-jobs:
-  choose:
-    runs-on: ubuntu-latest
-    timeout-minutes: 1
-    outputs:
-      config: ${{ steps.config.outputs.result }}
-    steps:
-      - uses: actions/github-script@v6
-        id: config
-        with:
-          script: |
-            if (`${context.repo.owner}/${context.repo.repo}` === 'ipfs/kubo') {
-              return {
-                labels: ['self-hosted', 'linux', 'x64', 'kubo'],
-                parallel: 10,
-                aws: true
-              }
-            } else {
-              return {
-                labels: ['ubuntu-latest'],
-                parallel: 3,
-                aws: false
-              }
-            }
-      - run: echo ${{ steps.config.outputs.result }}

.github/workflows/sharness.yml

@@ -3,6 +3,8 @@ name: Sharness
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  sharness-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   sharness-test:
-    needs: [sharness-runner]
-    runs-on: ${{ fromJSON(needs.sharness-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
@@ -26,7 +25,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - name: Checkout Kubo
         uses: actions/checkout@v3
         with:
@@ -57,7 +56,8 @@ jobs:
           TEST_EXPENSIVE: 1
           IPFS_CHECK_RCMGR_DEFAULTS: 1
           CONTINUE_ON_S_FAILURE: 1
-          PARALLEL: ${{ fromJSON(needs.sharness-runner.outputs.config).parallel }}
+          # increasing parallelism beyond 10 doesn't speed up the tests much
+          PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
       - name: Upload coverage report
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
         if: failure() || success()
@@ -86,12 +86,12 @@ jobs:
       - name: Upload one-page HTML report to S3
         id: one-page
         uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         with:
           source: kubo/test/sharness/test-results/sharness.html
           destination: sharness.html
       - name: Upload one-page HTML report
-        if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
+        if: github.repository != 'ipfs/kubo' && (failure() || success())
         uses: actions/upload-artifact@v3
         with:
           name: sharness.html
@@ -106,18 +106,18 @@ jobs:
       - name: Upload full HTML report to S3
         id: full
         uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         with:
           source: kubo/test/sharness/test-results/sharness-html
           destination: sharness-html/
       - name: Upload full HTML report
-        if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
+        if: github.repository != 'ipfs/kubo' && (failure() || success())
         uses: actions/upload-artifact@v3
         with:
           name: sharness-html
           path: kubo/test/sharness/test-results/sharness-html
       - name: Add S3 links to the summary
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         run: echo "$MD" >> $GITHUB_STEP_SUMMARY
         env:
           MD: |

.golangci.yml

@@ -4,5 +4,8 @@ linters:
 
 linters-settings:
   stylecheck:
+    checks:
+    - all
+    - '-ST1003'
     dot-import-whitelist:
       - github.com/ipfs/kubo/test/cli/testutils

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.21](docs/changelogs/v0.21.md)
 - [v0.20](docs/changelogs/v0.20.md)
 - [v0.19](docs/changelogs/v0.19.md)
 - [v0.18](docs/changelogs/v0.18.md)

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.19.1-buster
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.19-buster
 LABEL maintainer="Steven Allen <steven@stebalien.com>"
 
 ARG TARGETPLATFORM

README.md

@@ -187,10 +187,10 @@ $ ipfs get /ipns/dist.ipfs.tech/kubo/$VERSION/kubo_$VERSION_windows-amd64.zip
 With the purely functional package manager [Nix](https://nixos.org/nix/) you can install kubo (go-ipfs) like this:
 
 ```
-$ nix-env -i ipfs
+$ nix-env -i kubo
 ```
 
-You can also install the Package by using its attribute name, which is also `ipfs`.
+You can also install the Package by using its attribute name, which is also `kubo`.
 
 #### Solus
 
@@ -251,10 +251,10 @@ $ sudo port install ipfs
 In macOS you can use the purely functional package manager [Nix](https://nixos.org/nix/):
 
 ```
-$ nix-env -i ipfs
+$ nix-env -i kubo
 ```
 
-You can also install the Package by using its attribute name, which is also `ipfs`.
+You can also install the Package by using its attribute name, which is also `kubo`.
 
 #### Homebrew
 

client/rpc/README.md

@@ -0,0 +1,44 @@
+# `coreiface.CoreAPI` over http `rpc`
+
+> IPFS CoreAPI implementation using HTTP API
+
+This packages implements [`coreiface.CoreAPI`](https://pkg.go.dev/github.com/ipfs/boxo/coreiface#CoreAPI) over the HTTP API.
+
+## Documentation
+
+https://pkg.go.dev/github.com/ipfs/kubo/client/rpc
+
+### Example
+
+Pin file on your local IPFS node based on its CID:
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+
+    "github.com/ipfs/kubo/client/rpc"
+    path "github.com/ipfs/boxo/coreiface/path"
+)
+
+func main() {
+    // "Connect" to local node
+    node, err := rpc.NewLocalApi()
+    if err != nil {
+        fmt.Printf(err)
+        return
+    }
+    // Pin a given file by its CID
+    ctx := context.Background()
+    cid := "bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku"
+    p := path.New(cid)
+    err = node.Pin().Add(ctx, p)
+    if err != nil {
+    	fmt.Printf(err)
+        return
+    }
+    return
+}
+```

client/rpc/api.go

@@ -0,0 +1,230 @@
+package rpc
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	"github.com/ipfs/go-cid"
+	legacy "github.com/ipfs/go-ipld-legacy"
+	dagpb "github.com/ipld/go-codec-dagpb"
+	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/mitchellh/go-homedir"
+	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
+)
+
+const (
+	DefaultPathName = ".ipfs"
+	DefaultPathRoot = "~/" + DefaultPathName
+	DefaultApiFile  = "api"
+	EnvDir          = "IPFS_PATH"
+)
+
+// ErrApiNotFound if we fail to find a running daemon.
+var ErrApiNotFound = errors.New("ipfs api address could not be found")
+
+// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using
+// IPFS HTTP API.
+//
+// For interface docs see
+// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI
+type HttpApi struct {
+	url         string
+	httpcli     http.Client
+	Headers     http.Header
+	applyGlobal func(*requestBuilder)
+	ipldDecoder *legacy.Decoder
+}
+
+// NewLocalApi tries to construct new HttpApi instance communicating with local
+// IPFS daemon
+//
+// Daemon api address is pulled from the $IPFS_PATH/api file.
+// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs
+func NewLocalApi() (*HttpApi, error) {
+	baseDir := os.Getenv(EnvDir)
+	if baseDir == "" {
+		baseDir = DefaultPathRoot
+	}
+
+	return NewPathApi(baseDir)
+}
+
+// NewPathApi constructs new HttpApi by pulling api address from specified
+// ipfspath. Api file should be located at $ipfspath/api
+func NewPathApi(ipfspath string) (*HttpApi, error) {
+	a, err := ApiAddr(ipfspath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = ErrApiNotFound
+		}
+		return nil, err
+	}
+	return NewApi(a)
+}
+
+// ApiAddr reads api file in specified ipfs path
+func ApiAddr(ipfspath string) (ma.Multiaddr, error) {
+	baseDir, err := homedir.Expand(ipfspath)
+	if err != nil {
+		return nil, err
+	}
+
+	apiFile := filepath.Join(baseDir, DefaultApiFile)
+
+	api, err := os.ReadFile(apiFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return ma.NewMultiaddr(strings.TrimSpace(string(api)))
+}
+
+// NewApi constructs HttpApi with specified endpoint
+func NewApi(a ma.Multiaddr) (*HttpApi, error) {
+	c := &http.Client{
+		Transport: &http.Transport{
+			Proxy:             http.ProxyFromEnvironment,
+			DisableKeepAlives: true,
+		},
+	}
+
+	return NewApiWithClient(a, c)
+}
+
+// NewApiWithClient constructs HttpApi with specified endpoint and custom http client
+func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) {
+	_, url, err := manet.DialArgs(a)
+	if err != nil {
+		return nil, err
+	}
+
+	if a, err := ma.NewMultiaddr(url); err == nil {
+		_, host, err := manet.DialArgs(a)
+		if err == nil {
+			url = host
+		}
+	}
+
+	proto := "http://"
+
+	// By default, DialArgs is going to provide details suitable for connecting
+	// a socket to, but not really suitable for making an informed choice of http
+	// protocol.  For multiaddresses specifying tls and/or https we want to make
+	// a https request instead of a http request.
+	protocols := a.Protocols()
+	for _, p := range protocols {
+		if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS {
+			proto = "https://"
+			break
+		}
+	}
+
+	return NewURLApiWithClient(proto+url, c)
+}
+
+func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) {
+	decoder := legacy.NewDecoder()
+	// Add support for these codecs to match what is done in the merkledag library
+	// Note: to match prior behavior the go-ipld-prime CBOR decoder is manually included
+	// TODO: allow the codec registry used to be configured by the caller not through a global variable
+	decoder.RegisterCodec(cid.DagProtobuf, dagpb.Type.PBNode, merkledag.ProtoNodeConverter)
+	decoder.RegisterCodec(cid.Raw, basicnode.Prototype.Bytes, merkledag.RawNodeConverter)
+
+	api := &HttpApi{
+		url:         url,
+		httpcli:     *c,
+		Headers:     make(map[string][]string),
+		applyGlobal: func(*requestBuilder) {},
+		ipldDecoder: decoder,
+	}
+
+	// We don't support redirects.
+	api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
+		return fmt.Errorf("unexpected redirect")
+	}
+	return api, nil
+}
+
+func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) {
+	options, err := caopts.ApiOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	subApi := *api
+	subApi.applyGlobal = func(req *requestBuilder) {
+		if options.Offline {
+			req.Option("offline", options.Offline)
+		}
+	}
+
+	return &subApi, nil
+}
+
+func (api *HttpApi) Request(command string, args ...string) RequestBuilder {
+	headers := make(map[string]string)
+	if api.Headers != nil {
+		for k := range api.Headers {
+			headers[k] = api.Headers.Get(k)
+		}
+	}
+	return &requestBuilder{
+		command: command,
+		args:    args,
+		shell:   api,
+		headers: headers,
+	}
+}
+
+func (api *HttpApi) Unixfs() iface.UnixfsAPI {
+	return (*UnixfsAPI)(api)
+}
+
+func (api *HttpApi) Block() iface.BlockAPI {
+	return (*BlockAPI)(api)
+}
+
+func (api *HttpApi) Dag() iface.APIDagService {
+	return (*HttpDagServ)(api)
+}
+
+func (api *HttpApi) Name() iface.NameAPI {
+	return (*NameAPI)(api)
+}
+
+func (api *HttpApi) Key() iface.KeyAPI {
+	return (*KeyAPI)(api)
+}
+
+func (api *HttpApi) Pin() iface.PinAPI {
+	return (*PinAPI)(api)
+}
+
+func (api *HttpApi) Object() iface.ObjectAPI {
+	return (*ObjectAPI)(api)
+}
+
+func (api *HttpApi) Dht() iface.DhtAPI {
+	return (*DhtAPI)(api)
+}
+
+func (api *HttpApi) Swarm() iface.SwarmAPI {
+	return (*SwarmAPI)(api)
+}
+
+func (api *HttpApi) PubSub() iface.PubSubAPI {
+	return (*PubsubAPI)(api)
+}
+
+func (api *HttpApi) Routing() iface.RoutingAPI {
+	return (*RoutingAPI)(api)
+}

client/rpc/api_test.go

@@ -0,0 +1,162 @@
+package rpc
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/boxo/coreiface/tests"
+	"github.com/ipfs/kubo/test/cli/harness"
+	ma "github.com/multiformats/go-multiaddr"
+	"go.uber.org/multierr"
+)
+
+type NodeProvider struct{}
+
+func (np NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity, online bool, n int) ([]iface.CoreAPI, error) {
+	h := harness.NewT(t)
+
+	apis := make([]iface.CoreAPI, n)
+	nodes := h.NewNodes(n)
+
+	var wg, zero sync.WaitGroup
+	zeroNode := nodes[0]
+	wg.Add(len(apis))
+	zero.Add(1)
+
+	var errs []error
+	var errsLk sync.Mutex
+
+	for i, n := range nodes {
+		go func(i int, n *harness.Node) {
+			if err := func() error {
+				defer wg.Done()
+				var err error
+
+				n.Init("--empty-repo")
+
+				c := n.ReadConfig()
+				c.Experimental.FilestoreEnabled = true
+				n.WriteConfig(c)
+
+				n.StartDaemon("--enable-pubsub-experiment", "--offline="+strconv.FormatBool(!online))
+
+				if online {
+					if i > 0 {
+						zero.Wait()
+						n.Connect(zeroNode)
+					} else {
+						zero.Done()
+					}
+				}
+
+				apiMaddr, err := n.TryAPIAddr()
+				if err != nil {
+					return err
+				}
+
+				api, err := NewApi(apiMaddr)
+				if err != nil {
+					return err
+				}
+				apis[i] = api
+
+				// empty node is pinned even with --empty-repo, we don't want that
+				emptyNode := path.New("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+				if err := api.Pin().Rm(ctx, emptyNode); err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				errsLk.Lock()
+				errs = append(errs, err)
+				errsLk.Unlock()
+			}
+		}(i, n)
+	}
+
+	wg.Wait()
+
+	return apis, multierr.Combine(errs...)
+}
+
+func TestHttpApi(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS == "windows" {
+		t.Skip("skipping due to #9905")
+	}
+
+	tests.TestApi(NodeProvider{})(t)
+}
+
+func Test_NewURLApiWithClient_With_Headers(t *testing.T) {
+	t.Parallel()
+
+	var (
+		headerToTest        = "Test-Header"
+		expectedHeaderValue = "thisisaheadertest"
+	)
+	ts := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			val := r.Header.Get(headerToTest)
+			if val != expectedHeaderValue {
+				w.WriteHeader(400)
+				return
+			}
+			http.ServeContent(w, r, "", time.Now(), strings.NewReader("test"))
+		}),
+	)
+	defer ts.Close()
+	api, err := NewURLApiWithClient(ts.URL, &http.Client{
+		Transport: &http.Transport{
+			Proxy:             http.ProxyFromEnvironment,
+			DisableKeepAlives: true,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	api.Headers.Set(headerToTest, expectedHeaderValue)
+	if err := api.Pin().Rm(context.Background(), path.New("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) {
+	t.Parallel()
+
+	testcases := []struct {
+		address  string
+		expected string
+	}{
+		{address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"},
+		{address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"},
+		{address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"},
+		{address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"},
+	}
+
+	for _, tc := range testcases {
+		address, err := ma.NewMultiaddr(tc.address)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		api, err := NewApiWithClient(address, &http.Client{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if api.url != tc.expected {
+			t.Errorf("Expected = %s; got %s", tc.expected, api.url)
+		}
+	}
+}

client/rpc/apifile.go

@@ -0,0 +1,270 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/boxo/files"
+	unixfs "github.com/ipfs/boxo/ipld/unixfs"
+	"github.com/ipfs/go-cid"
+)
+
+const forwardSeekLimit = 1 << 14 //16k
+
+func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) {
+	if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS
+		var err error
+		p, err = api.core().ResolvePath(ctx, p)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var stat struct {
+		Hash string
+		Type string
+		Size int64 // unixfs size
+	}
+	err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat)
+	if err != nil {
+		return nil, err
+	}
+
+	switch stat.Type {
+	case "file":
+		return api.getFile(ctx, p, stat.Size)
+	case "directory":
+		return api.getDir(ctx, p, stat.Size)
+	default:
+		return nil, fmt.Errorf("unsupported file type '%s'", stat.Type)
+	}
+}
+
+type apiFile struct {
+	ctx  context.Context
+	core *HttpApi
+	size int64
+	path path.Path
+
+	r  *Response
+	at int64
+}
+
+func (f *apiFile) reset() error {
+	if f.r != nil {
+		_ = f.r.Cancel()
+		f.r = nil
+	}
+	req := f.core.Request("cat", f.path.String())
+	if f.at != 0 {
+		req.Option("offset", f.at)
+	}
+	resp, err := req.Send(f.ctx)
+	if err != nil {
+		return err
+	}
+	if resp.Error != nil {
+		return resp.Error
+	}
+	f.r = resp
+	return nil
+}
+
+func (f *apiFile) Read(p []byte) (int, error) {
+	n, err := f.r.Output.Read(p)
+	if n > 0 {
+		f.at += int64(n)
+	}
+	return n, err
+}
+
+func (f *apiFile) ReadAt(p []byte, off int64) (int, error) {
+	// Always make a new request. This method should be parallel-safe.
+	resp, err := f.core.Request("cat", f.path.String()).
+		Option("offset", off).Option("length", len(p)).Send(f.ctx)
+	if err != nil {
+		return 0, err
+	}
+	if resp.Error != nil {
+		return 0, resp.Error
+	}
+	defer resp.Output.Close()
+
+	n, err := io.ReadFull(resp.Output, p)
+	if err == io.ErrUnexpectedEOF {
+		err = io.EOF
+	}
+	return n, err
+}
+
+func (f *apiFile) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekEnd:
+		offset = f.size + offset
+	case io.SeekCurrent:
+		offset = f.at + offset
+	}
+	if f.at == offset { //noop
+		return offset, nil
+	}
+
+	if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip
+		r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at)
+
+		f.at += r
+		return f.at, err
+	}
+	f.at = offset
+	return f.at, f.reset()
+}
+
+func (f *apiFile) Close() error {
+	if f.r != nil {
+		return f.r.Cancel()
+	}
+	return nil
+}
+
+func (f *apiFile) Size() (int64, error) {
+	return f.size, nil
+}
+
+func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+	f := &apiFile{
+		ctx:  ctx,
+		core: api.core(),
+		size: size,
+		path: p,
+	}
+
+	return f, f.reset()
+}
+
+type apiIter struct {
+	ctx  context.Context
+	core *UnixfsAPI
+
+	err error
+
+	dec     *json.Decoder
+	curFile files.Node
+	cur     lsLink
+}
+
+func (it *apiIter) Err() error {
+	return it.err
+}
+
+func (it *apiIter) Name() string {
+	return it.cur.Name
+}
+
+func (it *apiIter) Next() bool {
+	if it.ctx.Err() != nil {
+		it.err = it.ctx.Err()
+		return false
+	}
+
+	var out lsOutput
+	if err := it.dec.Decode(&out); err != nil {
+		if err != io.EOF {
+			it.err = err
+		}
+		return false
+	}
+
+	if len(out.Objects) != 1 {
+		it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects))
+		return false
+	}
+
+	if len(out.Objects[0].Links) != 1 {
+		it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links))
+		return false
+	}
+
+	it.cur = out.Objects[0].Links[0]
+	c, err := cid.Parse(it.cur.Hash)
+	if err != nil {
+		it.err = err
+		return false
+	}
+
+	switch it.cur.Type {
+	case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory:
+		it.curFile, err = it.core.getDir(it.ctx, path.IpfsPath(c), int64(it.cur.Size))
+		if err != nil {
+			it.err = err
+			return false
+		}
+	case unixfs.TFile:
+		it.curFile, err = it.core.getFile(it.ctx, path.IpfsPath(c), int64(it.cur.Size))
+		if err != nil {
+			it.err = err
+			return false
+		}
+	default:
+		it.err = fmt.Errorf("file type %d not supported", it.cur.Type)
+		return false
+	}
+	return true
+}
+
+func (it *apiIter) Node() files.Node {
+	return it.curFile
+}
+
+type apiDir struct {
+	ctx  context.Context
+	core *UnixfsAPI
+	size int64
+	path path.Path
+
+	dec *json.Decoder
+}
+
+func (d *apiDir) Close() error {
+	return nil
+}
+
+func (d *apiDir) Size() (int64, error) {
+	return d.size, nil
+}
+
+func (d *apiDir) Entries() files.DirIterator {
+	return &apiIter{
+		ctx:  d.ctx,
+		core: d.core,
+		dec:  d.dec,
+	}
+}
+
+func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+	resp, err := api.core().Request("ls", p.String()).
+		Option("resolve-size", true).
+		Option("stream", true).Send(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	d := &apiDir{
+		ctx:  ctx,
+		core: api,
+		size: size,
+		path: p,
+
+		dec: json.NewDecoder(resp.Output),
+	}
+
+	return d, nil
+}
+
+var _ files.File = &apiFile{}
+var _ files.Directory = &apiDir{}

client/rpc/block.go

@@ -0,0 +1,134 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/go-cid"
+	mc "github.com/multiformats/go-multicodec"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BlockAPI HttpApi
+
+type blockStat struct {
+	Key   string
+	BSize int `json:"Size"`
+
+	cid cid.Cid
+}
+
+func (s *blockStat) Size() int {
+	return s.BSize
+}
+
+func (s *blockStat) Path() path.Resolved {
+	return path.IpldPath(s.cid)
+}
+
+func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) {
+	options, err := caopts.BlockPutOptions(opts...)
+	px := options.CidPrefix
+	if err != nil {
+		return nil, err
+	}
+
+	mht, ok := mh.Codes[px.MhType]
+	if !ok {
+		return nil, fmt.Errorf("unknowm mhType %d", px.MhType)
+	}
+
+	var cidOptKey, cidOptVal string
+	switch {
+	case px.Version == 0 && px.Codec == cid.DagProtobuf:
+		// ensure legacy --format=v0 passes as BlockPutOption still works
+		cidOptKey = "format"
+		cidOptVal = "v0"
+	default:
+		// pass codec as string
+		cidOptKey = "cid-codec"
+		cidOptVal = mc.Code(px.Codec).String()
+	}
+
+	req := api.core().Request("block/put").
+		Option("mhtype", mht).
+		Option("mhlen", px.MhLength).
+		Option(cidOptKey, cidOptVal).
+		Option("pin", options.Pin).
+		FileBody(r)
+
+	var out blockStat
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	out.cid, err = cid.Parse(out.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) {
+	resp, err := api.core().Request("block/get", p.String()).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, parseErrNotFoundWithFallbackToError(resp.Error)
+	}
+
+	//TODO: make get return ReadCloser to avoid copying
+	defer resp.Close()
+	b := new(bytes.Buffer)
+	if _, err := io.Copy(b, resp.Output); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error {
+	options, err := caopts.BlockRmOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	removedBlock := struct {
+		Hash  string `json:",omitempty"`
+		Error string `json:",omitempty"`
+	}{}
+
+	req := api.core().Request("block/rm").
+		Option("force", options.Force).
+		Arguments(p.String())
+
+	if err := req.Exec(ctx, &removedBlock); err != nil {
+		return err
+	}
+
+	return parseErrNotFoundWithFallbackToMSG(removedBlock.Error)
+}
+
+func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) {
+	var out blockStat
+	err := api.core().Request("block/stat", p.String()).Exec(ctx, &out)
+	if err != nil {
+		return nil, parseErrNotFoundWithFallbackToError(err)
+	}
+	out.cid, err = cid.Parse(out.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+func (api *BlockAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/dag.go

@@ -0,0 +1,136 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	format "github.com/ipfs/go-ipld-format"
+	multicodec "github.com/multiformats/go-multicodec"
+)
+
+type httpNodeAdder HttpApi
+type HttpDagServ httpNodeAdder
+type pinningHttpNodeAdder httpNodeAdder
+
+func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
+	r, err := api.core().Block().Get(ctx, path.IpldPath(c))
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	blk, err := blocks.NewBlockWithCid(data, c)
+	if err != nil {
+		return nil, err
+	}
+
+	return api.ipldDecoder.DecodeNode(ctx, blk)
+}
+
+func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption {
+	out := make(chan *format.NodeOption)
+
+	for _, c := range cids {
+		// TODO: Consider limiting concurrency of this somehow
+		go func(c cid.Cid) {
+			n, err := api.Get(ctx, c)
+
+			select {
+			case out <- &format.NodeOption{Node: n, Err: err}:
+			case <-ctx.Done():
+			}
+		}(c)
+	}
+	return out
+}
+
+func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error {
+	c := nd.Cid()
+	prefix := c.Prefix()
+
+	// preserve 'cid-codec' when sent over HTTP
+	cidCodec := multicodec.Code(prefix.Codec).String()
+
+	// 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80
+	// but we still support it here for backward-compatibility with use of CIDv0
+	format := ""
+	if prefix.Version == 0 {
+		cidCodec = ""
+		format = "v0"
+	}
+
+	stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()),
+		options.Block.Hash(prefix.MhType, prefix.MhLength),
+		options.Block.CidCodec(cidCodec),
+		options.Block.Format(format),
+		options.Block.Pin(pin))
+	if err != nil {
+		return err
+	}
+	if !stat.Path().Cid().Equals(c) {
+		return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String())
+	}
+	return nil
+}
+
+func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error {
+	for _, nd := range nds {
+		// TODO: optimize
+		if err := api.add(ctx, nd, pin); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error {
+	return (*httpNodeAdder)(api).addMany(ctx, nds, false)
+}
+
+func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error {
+	return (*httpNodeAdder)(api).add(ctx, nd, false)
+}
+
+func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error {
+	return (*httpNodeAdder)(api).add(ctx, nd, true)
+}
+
+func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error {
+	return (*httpNodeAdder)(api).addMany(ctx, nds, true)
+}
+
+func (api *HttpDagServ) Pinning() format.NodeAdder {
+	return (*pinningHttpNodeAdder)(api)
+}
+
+func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error {
+	return api.core().Block().Rm(ctx, path.IpldPath(c)) //TODO: should we force rm?
+}
+
+func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error {
+	for _, c := range cids {
+		// TODO: optimize
+		if err := api.Remove(ctx, c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (api *httpNodeAdder) core() *HttpApi {
+	return (*HttpApi)(api)
+}
+
+func (api *HttpDagServ) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/dht.go

@@ -0,0 +1,113 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/routing"
+)
+
+type DhtAPI HttpApi
+
+func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+	var out struct {
+		Type      routing.QueryEventType
+		Responses []peer.AddrInfo
+	}
+	resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx)
+	if err != nil {
+		return peer.AddrInfo{}, err
+	}
+	if resp.Error != nil {
+		return peer.AddrInfo{}, resp.Error
+	}
+	defer resp.Close()
+	dec := json.NewDecoder(resp.Output)
+	for {
+		if err := dec.Decode(&out); err != nil {
+			return peer.AddrInfo{}, err
+		}
+		if out.Type == routing.FinalPeer {
+			return out.Responses[0], nil
+		}
+	}
+}
+
+func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) {
+	options, err := caopts.DhtFindProvidersOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	rp, err := api.core().ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := api.core().Request("dht/findprovs", rp.Cid().String()).
+		Option("num-providers", options.NumProviders).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	res := make(chan peer.AddrInfo)
+
+	go func() {
+		defer resp.Close()
+		defer close(res)
+		dec := json.NewDecoder(resp.Output)
+
+		for {
+			var out struct {
+				Extra     string
+				Type      routing.QueryEventType
+				Responses []peer.AddrInfo
+			}
+
+			if err := dec.Decode(&out); err != nil {
+				return // todo: handle this somehow
+			}
+			if out.Type == routing.QueryError {
+				return // usually a 'not found' error
+				// todo: handle other errors
+			}
+			if out.Type == routing.Provider {
+				for _, pi := range out.Responses {
+					select {
+					case res <- pi:
+					case <-ctx.Done():
+						return
+					}
+				}
+			}
+		}
+	}()
+
+	return res, nil
+}
+
+func (api *DhtAPI) Provide(ctx context.Context, p path.Path, opts ...caopts.DhtProvideOption) error {
+	options, err := caopts.DhtProvideOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	rp, err := api.core().ResolvePath(ctx, p)
+	if err != nil {
+		return err
+	}
+
+	return api.core().Request("dht/provide", rp.Cid().String()).
+		Option("recursive", options.Recursive).
+		Exec(ctx, nil)
+}
+
+func (api *DhtAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/errors.go

@@ -0,0 +1,166 @@
+package rpc
+
+import (
+	"errors"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+// This file handle parsing and returning the correct ABI based errors from error messages
+
+type prePostWrappedNotFoundError struct {
+	pre  string
+	post string
+
+	wrapped ipld.ErrNotFound
+}
+
+func (e prePostWrappedNotFoundError) String() string {
+	return e.Error()
+}
+
+func (e prePostWrappedNotFoundError) Error() string {
+	return e.pre + e.wrapped.Error() + e.post
+}
+
+func (e prePostWrappedNotFoundError) Unwrap() error {
+	return e.wrapped
+}
+
+func parseErrNotFoundWithFallbackToMSG(msg string) error {
+	err, handled := parseErrNotFound(msg)
+	if handled {
+		return err
+	}
+
+	return errors.New(msg)
+}
+
+func parseErrNotFoundWithFallbackToError(msg error) error {
+	err, handled := parseErrNotFound(msg.Error())
+	if handled {
+		return err
+	}
+
+	return msg
+}
+
+func parseErrNotFound(msg string) (error, bool) {
+	if msg == "" {
+		return nil, true // Fast path
+	}
+
+	if err, handled := parseIPLDErrNotFound(msg); handled {
+		return err, true
+	}
+
+	if err, handled := parseBlockstoreNotFound(msg); handled {
+		return err, true
+	}
+
+	return nil, false
+}
+
+// Assume CIDs break on:
+// - Whitespaces: " \t\n\r\v\f"
+// - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr
+// - Double Quotes: "\"" this is for parsing %q and %#v formating
+const cidBreakSet = " \t\n\r\v\f;\""
+
+func parseIPLDErrNotFound(msg string) (error, bool) {
+	// The patern we search for is:
+	const ipldErrNotFoundKey = "ipld: could not find " /*CID*/
+	// We try to parse the CID, if it's invalid we give up and return a simple text error.
+	// We also accept "node" in place of the CID because that means it's an Undefined CID.
+
+	keyIndex := strings.Index(msg, ipldErrNotFoundKey)
+
+	if keyIndex < 0 { // Unknown error
+		return nil, false
+	}
+
+	cidStart := keyIndex + len(ipldErrNotFoundKey)
+
+	msgPostKey := msg[cidStart:]
+	var c cid.Cid
+	var postIndex int
+	if strings.HasPrefix(msgPostKey, "node") {
+		// Fallback case
+		c = cid.Undef
+		postIndex = len("node")
+	} else {
+		postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool {
+			return strings.ContainsAny(string(r), cidBreakSet)
+		})
+		if postIndex < 0 {
+			// no breakage meaning the string look like this something + "ipld: could not find bafy"
+			postIndex = len(msgPostKey)
+		}
+
+		cidStr := msgPostKey[:postIndex]
+
+		var err error
+		c, err = cid.Decode(cidStr)
+		if err != nil {
+			// failed to decode CID give up
+			return nil, false
+		}
+
+		// check that the CID is either a CIDv0 or a base32 multibase
+		// because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently
+		if c.Version() != 0 {
+			baseRune, _ := utf8.DecodeRuneInString(cidStr)
+			if baseRune == utf8.RuneError || baseRune != mbase.Base32 {
+				// not a multibase we expect, give up
+				return nil, false
+			}
+		}
+	}
+
+	err := ipld.ErrNotFound{Cid: c}
+	pre := msg[:keyIndex]
+	post := msgPostKey[postIndex:]
+
+	if len(pre) > 0 || len(post) > 0 {
+		return prePostWrappedNotFoundError{
+			pre:     pre,
+			post:    post,
+			wrapped: err,
+		}, true
+	}
+
+	return err, true
+}
+
+// This is a simple error type that just return msg as Error().
+// But that also match ipld.ErrNotFound when called with Is(err).
+// That is needed to keep compatiblity with code that use string.Contains(err.Error(), "blockstore: block not found")
+// and code using ipld.ErrNotFound
+type blockstoreNotFoundMatchingIPLDErrNotFound struct {
+	msg string
+}
+
+func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string {
+	return e.Error()
+}
+
+func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string {
+	return e.msg
+}
+
+func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool {
+	_, ok := err.(ipld.ErrNotFound)
+	return ok
+}
+
+func parseBlockstoreNotFound(msg string) (error, bool) {
+	if !strings.Contains(msg, "blockstore: block not found") {
+		return nil, false
+	}
+
+	return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true
+}

client/rpc/errors_test.go

@@ -0,0 +1,99 @@
+package rpc
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	mbase "github.com/multiformats/go-multibase"
+	mh "github.com/multiformats/go-multihash"
+)
+
+var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2}
+
+func doParseIpldNotFoundTest(t *testing.T, original error) {
+	originalMsg := original.Error()
+
+	rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg)
+
+	rebuiltMsg := rebuilt.Error()
+
+	if originalMsg != rebuiltMsg {
+		t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg)
+	}
+
+	originalNotFound := ipld.IsNotFound(original)
+	rebuiltNotFound := ipld.IsNotFound(rebuilt)
+	if originalNotFound != rebuiltNotFound {
+		t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound)
+	}
+}
+
+func TestParseIPLDNotFound(t *testing.T) {
+	t.Parallel()
+
+	if err := parseErrNotFoundWithFallbackToMSG(""); err != nil {
+		t.Errorf("expected empty string to give no error; got %T %q", err, err.Error())
+	}
+
+	cidBreaks := make([]string, len(cidBreakSet))
+	for i, v := range cidBreakSet {
+		cidBreaks[i] = "%w" + string(v)
+	}
+
+	base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC)
+	if err != nil {
+		t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error())
+	}
+
+	for _, wrap := range append(cidBreaks,
+		"",
+		"merkledag: %w",
+		"testing: %w the test",
+		"%w is wrong",
+	) {
+		for _, err := range [...]error{
+			errors.New("ipld: could not find "),
+			errors.New("ipld: could not find Bad_CID"),
+			errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs
+			errors.New("network connection timeout"),
+			ipld.ErrNotFound{Cid: cid.Undef},
+			ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)},
+			ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)},
+		} {
+			if wrap != "" {
+				err = fmt.Errorf(wrap, err)
+			}
+
+			doParseIpldNotFoundTest(t, err)
+		}
+	}
+}
+
+func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) {
+	t.Parallel()
+
+	if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) {
+		t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false")
+	}
+
+	for _, wrap := range [...]string{
+		"",
+		"merkledag: %w",
+		"testing: %w the test",
+		"%w is wrong",
+	} {
+		for _, err := range [...]error{
+			errors.New("network connection timeout"),
+			blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"},
+		} {
+			if wrap != "" {
+				err = fmt.Errorf(wrap, err)
+			}
+
+			doParseIpldNotFoundTest(t, err)
+		}
+	}
+}

client/rpc/key.go

@@ -0,0 +1,123 @@
+package rpc
+
+import (
+	"context"
+	"errors"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+type KeyAPI HttpApi
+
+type keyOutput struct {
+	JName string `json:"Name"`
+	Id    string
+
+	pid peer.ID
+}
+
+func (k *keyOutput) Name() string {
+	return k.JName
+}
+
+func (k *keyOutput) Path() path.Path {
+	return path.New("/ipns/" + k.Id)
+}
+
+func (k *keyOutput) ID() peer.ID {
+	return k.pid
+}
+
+func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) {
+	options, err := caopts.KeyGenerateOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out keyOutput
+	err = api.core().Request("key/gen", name).
+		Option("type", options.Algorithm).
+		Option("size", options.Size).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+	out.pid, err = peer.Decode(out.Id)
+	return &out, err
+}
+
+func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) {
+	options, err := caopts.KeyRenameOptions(opts...)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var out struct {
+		Was       string
+		Now       string
+		Id        string
+		Overwrite bool
+	}
+	err = api.core().Request("key/rename", oldName, newName).
+		Option("force", options.Force).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, false, err
+	}
+
+	id := &keyOutput{JName: out.Now, Id: out.Id}
+	id.pid, err = peer.Decode(id.Id)
+	return id, out.Overwrite, err
+}
+
+func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) {
+	var out struct{ Keys []*keyOutput }
+	if err := api.core().Request("key/list").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]iface.Key, len(out.Keys))
+	for i, k := range out.Keys {
+		var err error
+		k.pid, err = peer.Decode(k.Id)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = k
+	}
+
+	return res, nil
+}
+
+func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) {
+	var id struct{ ID string }
+	if err := api.core().Request("id").Exec(ctx, &id); err != nil {
+		return nil, err
+	}
+
+	var err error
+	out := keyOutput{JName: "self", Id: id.ID}
+	out.pid, err = peer.Decode(out.Id)
+	return &out, err
+}
+
+func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) {
+	var out struct{ Keys []keyOutput }
+	if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	if len(out.Keys) != 1 {
+		return nil, errors.New("got unexpected number of keys back")
+	}
+
+	var err error
+	out.Keys[0].pid, err = peer.Decode(out.Keys[0].Id)
+	return &out.Keys[0], err
+}
+
+func (api *KeyAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/name.go

@@ -0,0 +1,140 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
+	"github.com/ipfs/boxo/coreiface/path"
+)
+
+type NameAPI HttpApi
+
+type ipnsEntry struct {
+	JName  string `json:"Name"`
+	JValue string `json:"Value"`
+
+	path path.Path
+}
+
+func (e *ipnsEntry) Name() string {
+	return e.JName
+}
+
+func (e *ipnsEntry) Value() path.Path {
+	return e.path
+}
+
+func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) {
+	options, err := caopts.NamePublishOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	req := api.core().Request("name/publish", p.String()).
+		Option("key", options.Key).
+		Option("allow-offline", options.AllowOffline).
+		Option("lifetime", options.ValidTime).
+		Option("resolve", false)
+
+	if options.TTL != nil {
+		req.Option("ttl", options.TTL)
+	}
+
+	var out ipnsEntry
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	out.path = path.New(out.JValue)
+	return &out, out.path.IsValid()
+}
+
+func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) {
+	options, err := caopts.NameResolveOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	ropts := nsopts.ProcessOpts(options.ResolveOpts)
+	if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 {
+		return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit)
+	}
+
+	req := api.core().Request("name/resolve", name).
+		Option("nocache", !options.Cache).
+		Option("recursive", ropts.Depth != 1).
+		Option("dht-record-count", ropts.DhtRecordCount).
+		Option("dht-timeout", ropts.DhtTimeout).
+		Option("stream", true)
+	resp, err := req.Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	res := make(chan iface.IpnsResult)
+
+	go func() {
+		defer close(res)
+		defer resp.Close()
+
+		dec := json.NewDecoder(resp.Output)
+
+		for {
+			var out struct{ Path string }
+			err := dec.Decode(&out)
+			if err == io.EOF {
+				return
+			}
+			var ires iface.IpnsResult
+			if err == nil {
+				ires.Path = path.New(out.Path)
+			}
+
+			select {
+			case res <- ires:
+			case <-ctx.Done():
+			}
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	return res, nil
+}
+
+func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) {
+	options, err := caopts.NameResolveOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	ropts := nsopts.ProcessOpts(options.ResolveOpts)
+	if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 {
+		return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit)
+	}
+
+	req := api.core().Request("name/resolve", name).
+		Option("nocache", !options.Cache).
+		Option("recursive", ropts.Depth != 1).
+		Option("dht-record-count", ropts.DhtRecordCount).
+		Option("dht-timeout", ropts.DhtTimeout)
+
+	var out struct{ Path string }
+	if err := req.Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	return path.New(out.Path), nil
+}
+
+func (api *NameAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/object.go

@@ -0,0 +1,260 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	ft "github.com/ipfs/boxo/ipld/unixfs"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+type ObjectAPI HttpApi
+
+type objectOut struct {
+	Hash string
+}
+
+func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) {
+	options, err := caopts.ObjectNewOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var n ipld.Node
+	switch options.Type {
+	case "empty":
+		n = new(merkledag.ProtoNode)
+	case "unixfs-dir":
+		n = ft.EmptyDirNode()
+	default:
+		return nil, fmt.Errorf("unknown object type: %s", options.Type)
+	}
+
+	return n, nil
+}
+
+func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (path.Resolved, error) {
+	options, err := caopts.ObjectPutOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out objectOut
+	err = api.core().Request("object/put").
+		Option("inputenc", options.InputEnc).
+		Option("datafieldenc", options.DataType).
+		Option("pin", options.Pin).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) {
+	r, err := api.core().Block().Get(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return merkledag.DecodeProtobuf(b)
+}
+
+func (api *ObjectAPI) Data(ctx context.Context, p path.Path) (io.Reader, error) {
+	resp, err := api.core().Request("object/data", p.String()).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	//TODO: make Data return ReadCloser to avoid copying
+	defer resp.Close()
+	b := new(bytes.Buffer)
+	if _, err := io.Copy(b, resp.Output); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (api *ObjectAPI) Links(ctx context.Context, p path.Path) ([]*ipld.Link, error) {
+	var out struct {
+		Links []struct {
+			Name string
+			Hash string
+			Size uint64
+		}
+	}
+	if err := api.core().Request("object/links", p.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := make([]*ipld.Link, len(out.Links))
+	for i, l := range out.Links {
+		c, err := cid.Parse(l.Hash)
+		if err != nil {
+			return nil, err
+		}
+
+		res[i] = &ipld.Link{
+			Cid:  c,
+			Name: l.Name,
+			Size: l.Size,
+		}
+	}
+
+	return res, nil
+}
+
+func (api *ObjectAPI) Stat(ctx context.Context, p path.Path) (*iface.ObjectStat, error) {
+	var out struct {
+		Hash           string
+		NumLinks       int
+		BlockSize      int
+		LinksSize      int
+		DataSize       int
+		CumulativeSize int
+	}
+	if err := api.core().Request("object/stat", p.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return &iface.ObjectStat{
+		Cid:            c,
+		NumLinks:       out.NumLinks,
+		BlockSize:      out.BlockSize,
+		LinksSize:      out.LinksSize,
+		DataSize:       out.DataSize,
+		CumulativeSize: out.CumulativeSize,
+	}, nil
+}
+
+func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.Resolved, error) {
+	options, err := caopts.ObjectAddLinkOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out objectOut
+	err = api.core().Request("object/patch/add-link", base.String(), name, child.String()).
+		Option("create", options.Create).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) {
+	var out objectOut
+	err := api.core().Request("object/patch/rm-link", base.String(), link).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+func (api *ObjectAPI) AppendData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) {
+	var out objectOut
+	err := api.core().Request("object/patch/append-data", p.String()).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+func (api *ObjectAPI) SetData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) {
+	var out objectOut
+	err := api.core().Request("object/patch/set-data", p.String()).
+		FileBody(r).
+		Exec(ctx, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+type change struct {
+	Type   iface.ChangeType
+	Path   string
+	Before cid.Cid
+	After  cid.Cid
+}
+
+func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) {
+	var out struct {
+		Changes []change
+	}
+	if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := make([]iface.ObjectChange, len(out.Changes))
+	for i, ch := range out.Changes {
+		res[i] = iface.ObjectChange{
+			Type: ch.Type,
+			Path: ch.Path,
+		}
+		if ch.Before != cid.Undef {
+			res[i].Before = path.IpfsPath(ch.Before)
+		}
+		if ch.After != cid.Undef {
+			res[i].After = path.IpfsPath(ch.After)
+		}
+	}
+	return res, nil
+}
+
+func (api *ObjectAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/path.go

@@ -0,0 +1,52 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/ipfs/boxo/coreiface/path"
+	ipfspath "github.com/ipfs/boxo/path"
+	cid "github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) {
+	var out struct {
+		Cid     cid.Cid
+		RemPath string
+	}
+
+	//TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help
+
+	var err error
+	if p.Namespace() == "ipns" {
+		if p, err = api.Name().Resolve(ctx, p.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	// TODO:
+	ipath, err := ipfspath.FromSegments("/"+p.Namespace()+"/", out.Cid.String(), out.RemPath)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := cid.Parse(ipfspath.Path(p.String()).Segments()[1])
+	if err != nil {
+		return nil, err
+	}
+
+	return path.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil
+}
+
+func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return api.Dag().Get(ctx, rp.Cid())
+}

client/rpc/pin.go

@@ -0,0 +1,266 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"strings"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/go-cid"
+	"github.com/pkg/errors"
+)
+
+type PinAPI HttpApi
+
+type pinRefKeyObject struct {
+	Type string
+}
+
+type pinRefKeyList struct {
+	Keys map[string]pinRefKeyObject
+}
+
+type pin struct {
+	path path.Resolved
+	typ  string
+	err  error
+}
+
+func (p pin) Err() error {
+	return p.err
+}
+
+func (p pin) Path() path.Resolved {
+	return p.path
+}
+
+func (p pin) Type() string {
+	return p.typ
+}
+
+func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error {
+	options, err := caopts.PinAddOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().Request("pin/add", p.String()).
+		Option("recursive", options.Recursive).Exec(ctx, nil)
+}
+
+type pinLsObject struct {
+	Cid  string
+	Type string
+}
+
+func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) {
+	options, err := caopts.PinLsOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := api.core().Request("pin/ls").
+		Option("type", options.Type).
+		Option("stream", true).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	pins := make(chan iface.Pin)
+	go func(ch chan<- iface.Pin) {
+		defer res.Output.Close()
+		defer close(ch)
+
+		dec := json.NewDecoder(res.Output)
+		var out pinLsObject
+		for {
+			switch err := dec.Decode(&out); err {
+			case nil:
+			case io.EOF:
+				return
+			default:
+				select {
+				case ch <- pin{err: err}:
+					return
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			c, err := cid.Parse(out.Cid)
+			if err != nil {
+				select {
+				case ch <- pin{err: err}:
+					return
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			select {
+			case ch <- pin{typ: out.Type, path: path.IpldPath(c)}:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}(pins)
+	return pins, nil
+}
+
+// IsPinned returns whether or not the given cid is pinned
+// and an explanation of why its pinned
+func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) {
+	options, err := caopts.PinIsPinnedOptions(opts...)
+	if err != nil {
+		return "", false, err
+	}
+	var out pinRefKeyList
+	err = api.core().Request("pin/ls").
+		Option("type", options.WithType).
+		Option("arg", p.String()).
+		Exec(ctx, &out)
+	if err != nil {
+		// TODO: This error-type discrimination based on sub-string matching is brittle.
+		// It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563
+		if strings.Contains(err.Error(), "is not pinned") {
+			return "", false, nil
+		}
+		return "", false, err
+	}
+
+	for _, obj := range out.Keys {
+		return obj.Type, true, nil
+	}
+	return "", false, errors.New("http api returned no error and no results")
+}
+
+func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error {
+	options, err := caopts.PinRmOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().Request("pin/rm", p.String()).
+		Option("recursive", options.Recursive).
+		Exec(ctx, nil)
+}
+
+func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error {
+	options, err := caopts.PinUpdateOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	return api.core().Request("pin/update", from.String(), to.String()).
+		Option("unpin", options.Unpin).Exec(ctx, nil)
+}
+
+type pinVerifyRes struct {
+	ok       bool
+	badNodes []iface.BadPinNode
+	err      error
+}
+
+func (r pinVerifyRes) Ok() bool {
+	return r.ok
+}
+
+func (r pinVerifyRes) BadNodes() []iface.BadPinNode {
+	return r.badNodes
+}
+
+func (r pinVerifyRes) Err() error {
+	return r.err
+}
+
+type badNode struct {
+	err error
+	cid cid.Cid
+}
+
+func (n badNode) Path() path.Resolved {
+	return path.IpldPath(n.cid)
+}
+
+func (n badNode) Err() error {
+	return n.err
+}
+
+func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) {
+	resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	res := make(chan iface.PinStatus)
+
+	go func() {
+		defer resp.Close()
+		defer close(res)
+		dec := json.NewDecoder(resp.Output)
+		for {
+			var out struct {
+				Cid string
+				Err string
+				Ok  bool
+
+				BadNodes []struct {
+					Cid string
+					Err string
+				}
+			}
+			if err := dec.Decode(&out); err != nil {
+				if err == io.EOF {
+					return
+				}
+				select {
+				case res <- pinVerifyRes{err: err}:
+					return
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			if out.Err != "" {
+				select {
+				case res <- pinVerifyRes{err: errors.New(out.Err)}:
+					return
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			badNodes := make([]iface.BadPinNode, len(out.BadNodes))
+			for i, n := range out.BadNodes {
+				c, err := cid.Decode(n.Cid)
+				if err != nil {
+					badNodes[i] = badNode{cid: c, err: err}
+					continue
+				}
+
+				if n.Err != "" {
+					err = errors.New(n.Err)
+				}
+				badNodes[i] = badNode{cid: c, err: err}
+			}
+
+			select {
+			case res <- pinVerifyRes{ok: out.Ok, badNodes: badNodes}:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return res, nil
+}
+
+func (api *PinAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/pubsub.go

@@ -0,0 +1,214 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/libp2p/go-libp2p/core/peer"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+type PubsubAPI HttpApi
+
+func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	topics := make([]string, len(out.Strings))
+	for n, mb := range out.Strings {
+		_, topic, err := mbase.Decode(mb)
+		if err != nil {
+			return nil, err
+		}
+		topics[n] = string(topic)
+	}
+	return topics, nil
+}
+
+func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) {
+	options, err := caopts.PubSubPeersOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var out struct {
+		Strings []string
+	}
+
+	var optionalTopic string
+	if len(options.Topic) > 0 {
+		optionalTopic = toMultibase([]byte(options.Topic))
+	}
+	if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]peer.ID, len(out.Strings))
+	for i, sid := range out.Strings {
+		id, err := peer.Decode(sid)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = id
+	}
+	return res, nil
+}
+
+func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error {
+	return api.core().Request("pubsub/pub", toMultibase([]byte(topic))).
+		FileBody(bytes.NewReader(message)).
+		Exec(ctx, nil)
+}
+
+type pubsubSub struct {
+	messages chan pubsubMessage
+
+	done    chan struct{}
+	rcloser func() error
+}
+
+type pubsubMessage struct {
+	JFrom     string   `json:"from,omitempty"`
+	JData     string   `json:"data,omitempty"`
+	JSeqno    string   `json:"seqno,omitempty"`
+	JTopicIDs []string `json:"topicIDs,omitempty"`
+
+	// real values after unpacking from text/multibase envelopes
+	from   peer.ID
+	data   []byte
+	seqno  []byte
+	topics []string
+
+	err error
+}
+
+func (msg *pubsubMessage) From() peer.ID {
+	return msg.from
+}
+
+func (msg *pubsubMessage) Data() []byte {
+	return msg.data
+}
+
+func (msg *pubsubMessage) Seq() []byte {
+	return msg.seqno
+}
+
+// TODO: do we want to keep this interface as []string,
+// or change to more correct [][]byte?
+func (msg *pubsubMessage) Topics() []string {
+	return msg.topics
+}
+
+func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) {
+	select {
+	case msg, ok := <-s.messages:
+		if !ok {
+			return nil, io.EOF
+		}
+		if msg.err != nil {
+			return nil, msg.err
+		}
+		// unpack values from text/multibase envelopes
+		var err error
+		msg.from, err = peer.Decode(msg.JFrom)
+		if err != nil {
+			return nil, err
+		}
+		_, msg.data, err = mbase.Decode(msg.JData)
+		if err != nil {
+			return nil, err
+		}
+		_, msg.seqno, err = mbase.Decode(msg.JSeqno)
+		if err != nil {
+			return nil, err
+		}
+		for _, mbt := range msg.JTopicIDs {
+			_, topic, err := mbase.Decode(mbt)
+			if err != nil {
+				return nil, err
+			}
+			msg.topics = append(msg.topics, string(topic))
+		}
+		return &msg, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) {
+	/* right now we have no options (discover got deprecated)
+	options, err := caopts.PubSubSubscribeOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+	*/
+	resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	sub := &pubsubSub{
+		messages: make(chan pubsubMessage),
+		done:     make(chan struct{}),
+		rcloser: func() error {
+			return resp.Cancel()
+		},
+	}
+
+	dec := json.NewDecoder(resp.Output)
+
+	go func() {
+		defer close(sub.messages)
+
+		for {
+			var msg pubsubMessage
+			if err := dec.Decode(&msg); err != nil {
+				if err == io.EOF {
+					return
+				}
+				msg.err = err
+			}
+
+			select {
+			case sub.messages <- msg:
+			case <-sub.done:
+				return
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return sub, nil
+}
+
+func (s *pubsubSub) Close() error {
+	if s.done != nil {
+		close(s.done)
+		s.done = nil
+	}
+	return s.rcloser()
+}
+
+func (api *PubsubAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}
+
+// Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body)
+func toMultibase(data []byte) string {
+	mb, _ := mbase.Encode(mbase.Base64url, data)
+	return mb
+}

client/rpc/request.go

@@ -0,0 +1,36 @@
+package rpc
+
+import (
+	"context"
+	"io"
+	"strings"
+)
+
+type Request struct {
+	Ctx     context.Context
+	ApiBase string
+	Command string
+	Args    []string
+	Opts    map[string]string
+	Body    io.Reader
+	Headers map[string]string
+}
+
+func NewRequest(ctx context.Context, url, command string, args ...string) *Request {
+	if !strings.HasPrefix(url, "http") {
+		url = "http://" + url
+	}
+
+	opts := map[string]string{
+		"encoding":        "json",
+		"stream-channels": "true",
+	}
+	return &Request{
+		Ctx:     ctx,
+		ApiBase: url + "/api/v0",
+		Command: command,
+		Args:    args,
+		Opts:    opts,
+		Headers: make(map[string]string),
+	}
+}

client/rpc/requestbuilder.go

@@ -0,0 +1,127 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/ipfs/boxo/files"
+)
+
+type RequestBuilder interface {
+	Arguments(args ...string) RequestBuilder
+	BodyString(body string) RequestBuilder
+	BodyBytes(body []byte) RequestBuilder
+	Body(body io.Reader) RequestBuilder
+	FileBody(body io.Reader) RequestBuilder
+	Option(key string, value interface{}) RequestBuilder
+	Header(name, value string) RequestBuilder
+	Send(ctx context.Context) (*Response, error)
+	Exec(ctx context.Context, res interface{}) error
+}
+
+// requestBuilder is an IPFS commands request builder.
+type requestBuilder struct {
+	command string
+	args    []string
+	opts    map[string]string
+	headers map[string]string
+	body    io.Reader
+
+	shell *HttpApi
+}
+
+// Arguments adds the arguments to the args.
+func (r *requestBuilder) Arguments(args ...string) RequestBuilder {
+	r.args = append(r.args, args...)
+	return r
+}
+
+// BodyString sets the request body to the given string.
+func (r *requestBuilder) BodyString(body string) RequestBuilder {
+	return r.Body(strings.NewReader(body))
+}
+
+// BodyBytes sets the request body to the given buffer.
+func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder {
+	return r.Body(bytes.NewReader(body))
+}
+
+// Body sets the request body to the given reader.
+func (r *requestBuilder) Body(body io.Reader) RequestBuilder {
+	r.body = body
+	return r
+}
+
+// FileBody sets the request body to the given reader wrapped into multipartreader.
+func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder {
+	pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil)
+	d := files.NewMapDirectory(map[string]files.Node{"": pr})
+	r.body = files.NewMultiFileReader(d, false)
+
+	return r
+}
+
+// Option sets the given option.
+func (r *requestBuilder) Option(key string, value interface{}) RequestBuilder {
+	var s string
+	switch v := value.(type) {
+	case bool:
+		s = strconv.FormatBool(v)
+	case string:
+		s = v
+	case []byte:
+		s = string(v)
+	default:
+		// slow case.
+		s = fmt.Sprint(value)
+	}
+	if r.opts == nil {
+		r.opts = make(map[string]string, 1)
+	}
+	r.opts[key] = s
+	return r
+}
+
+// Header sets the given header.
+func (r *requestBuilder) Header(name, value string) RequestBuilder {
+	if r.headers == nil {
+		r.headers = make(map[string]string, 1)
+	}
+	r.headers[name] = value
+	return r
+}
+
+// Send sends the request and return the response.
+func (r *requestBuilder) Send(ctx context.Context) (*Response, error) {
+	r.shell.applyGlobal(r)
+
+	req := NewRequest(ctx, r.shell.url, r.command, r.args...)
+	req.Opts = r.opts
+	req.Headers = r.headers
+	req.Body = r.body
+	return req.Send(&r.shell.httpcli)
+}
+
+// Exec sends the request a request and decodes the response.
+func (r *requestBuilder) Exec(ctx context.Context, res interface{}) error {
+	httpRes, err := r.Send(ctx)
+	if err != nil {
+		return err
+	}
+
+	if res == nil {
+		lateErr := httpRes.Close()
+		if httpRes.Error != nil {
+			return httpRes.Error
+		}
+		return lateErr
+	}
+
+	return httpRes.decode(res)
+}
+
+var _ RequestBuilder = &requestBuilder{}

client/rpc/response.go

@@ -0,0 +1,170 @@
+package rpc
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/ipfs/boxo/files"
+	cmds "github.com/ipfs/go-ipfs-cmds"
+	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
+)
+
+type Error = cmds.Error
+
+type trailerReader struct {
+	resp *http.Response
+}
+
+func (r *trailerReader) Read(b []byte) (int, error) {
+	n, err := r.resp.Body.Read(b)
+	if err != nil {
+		if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" {
+			err = errors.New(e)
+		}
+	}
+	return n, err
+}
+
+func (r *trailerReader) Close() error {
+	return r.resp.Body.Close()
+}
+
+type Response struct {
+	Output io.ReadCloser
+	Error  *Error
+}
+
+func (r *Response) Close() error {
+	if r.Output != nil {
+
+		// drain output (response body)
+		_, err1 := io.Copy(io.Discard, r.Output)
+		err2 := r.Output.Close()
+		if err1 != nil {
+			return err1
+		}
+		return err2
+	}
+	return nil
+}
+
+// Cancel aborts running request (without draining request body)
+func (r *Response) Cancel() error {
+	if r.Output != nil {
+		return r.Output.Close()
+	}
+
+	return nil
+}
+
+// Decode reads request body and decodes it as json
+func (r *Response) decode(dec interface{}) error {
+	if r.Error != nil {
+		return r.Error
+	}
+
+	err := json.NewDecoder(r.Output).Decode(dec)
+	err2 := r.Close()
+	if err != nil {
+		return err
+	}
+
+	return err2
+}
+
+func (r *Request) Send(c *http.Client) (*Response, error) {
+	url := r.getURL()
+	req, err := http.NewRequest("POST", url, r.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	req = req.WithContext(r.Ctx)
+
+	// Add any headers that were supplied via the requestBuilder.
+	for k, v := range r.Headers {
+		req.Header.Add(k, v)
+	}
+
+	if fr, ok := r.Body.(*files.MultiFileReader); ok {
+		req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary())
+		req.Header.Set("Content-Disposition", "form-data; name=\"files\"")
+	}
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, err
+	}
+
+	nresp := new(Response)
+
+	nresp.Output = &trailerReader{resp}
+	if resp.StatusCode >= http.StatusBadRequest {
+		e := new(Error)
+		switch {
+		case resp.StatusCode == http.StatusNotFound:
+			e.Message = "command not found"
+		case contentType == "text/plain":
+			out, err := io.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err)
+			}
+			e.Message = string(out)
+
+			// set special status codes.
+			switch resp.StatusCode {
+			case http.StatusNotFound, http.StatusBadRequest:
+				e.Code = cmds.ErrClient
+			case http.StatusTooManyRequests:
+				e.Code = cmds.ErrRateLimited
+			case http.StatusForbidden:
+				e.Code = cmds.ErrForbidden
+			}
+		case contentType == "application/json":
+			if err = json.NewDecoder(resp.Body).Decode(e); err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err)
+			}
+		default:
+			// This is a server-side bug (probably).
+			e.Code = cmds.ErrImplementation
+			fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType)
+			out, err := io.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err)
+			}
+			e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out)
+		}
+		nresp.Error = e
+		nresp.Output = nil
+
+		// drain body and close
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}
+
+	return nresp, nil
+}
+
+func (r *Request) getURL() string {
+
+	values := make(url.Values)
+	for _, arg := range r.Args {
+		values.Add("arg", arg)
+	}
+	for k, v := range r.Opts {
+		values.Add(k, v)
+	}
+
+	return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode())
+}

client/rpc/routing.go

@@ -0,0 +1,64 @@
+package rpc
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+
+	"github.com/ipfs/boxo/coreiface/options"
+	"github.com/libp2p/go-libp2p/core/routing"
+)
+
+type RoutingAPI HttpApi
+
+func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) {
+	resp, err := api.core().Request("routing/get", key).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	defer resp.Close()
+
+	var out routing.QueryEvent
+
+	dec := json.NewDecoder(resp.Output)
+	if err := dec.Decode(&out); err != nil {
+		return nil, err
+	}
+
+	res, err := base64.StdEncoding.DecodeString(out.Extra)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte, opts ...options.RoutingPutOption) error {
+	var cfg options.RoutingPutSettings
+	for _, o := range opts {
+		if err := o(&cfg); err != nil {
+			return err
+		}
+	}
+
+	resp, err := api.core().Request("routing/put", key).
+		Option("allow-offline", cfg.AllowOffline).
+		FileBody(bytes.NewReader(value)).
+		Send(ctx)
+
+	if err != nil {
+		return err
+	}
+	if resp.Error != nil {
+		return resp.Error
+	}
+	return nil
+}
+
+func (api *RoutingAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/swarm.go

@@ -0,0 +1,187 @@
+package rpc
+
+import (
+	"context"
+	"time"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/protocol"
+	"github.com/multiformats/go-multiaddr"
+)
+
+type SwarmAPI HttpApi
+
+func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error {
+	pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty())
+	if err != nil {
+		return err
+	}
+
+	saddrs := make([]string, len(pi.Addrs))
+	for i, addr := range pi.Addrs {
+		saddrs[i] = addr.Encapsulate(pidma).String()
+	}
+
+	return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil)
+}
+
+func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error {
+	return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil)
+}
+
+type connInfo struct {
+	addr      multiaddr.Multiaddr
+	peer      peer.ID
+	latency   time.Duration
+	muxer     string
+	direction network.Direction
+	streams   []protocol.ID
+}
+
+func (c *connInfo) ID() peer.ID {
+	return c.peer
+}
+
+func (c *connInfo) Address() multiaddr.Multiaddr {
+	return c.addr
+}
+
+func (c *connInfo) Direction() network.Direction {
+	return c.direction
+}
+
+func (c *connInfo) Latency() (time.Duration, error) {
+	return c.latency, nil
+}
+
+func (c *connInfo) Streams() ([]protocol.ID, error) {
+	return c.streams, nil
+}
+
+func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) {
+	var resp struct {
+		Peers []struct {
+			Addr      string
+			Peer      string
+			Latency   string
+			Muxer     string
+			Direction network.Direction
+			Streams   []struct {
+				Protocol string
+			}
+		}
+	}
+
+	err := api.core().Request("swarm/peers").
+		Option("streams", true).
+		Option("latency", true).
+		Exec(ctx, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]iface.ConnectionInfo, len(resp.Peers))
+	for i, conn := range resp.Peers {
+		latency, _ := time.ParseDuration(conn.Latency)
+		out := &connInfo{
+			latency:   latency,
+			muxer:     conn.Muxer,
+			direction: conn.Direction,
+		}
+
+		out.peer, err = peer.Decode(conn.Peer)
+		if err != nil {
+			return nil, err
+		}
+
+		out.addr, err = multiaddr.NewMultiaddr(conn.Addr)
+		if err != nil {
+			return nil, err
+		}
+
+		out.streams = make([]protocol.ID, len(conn.Streams))
+		for i, p := range conn.Streams {
+			out.streams[i] = protocol.ID(p.Protocol)
+		}
+
+		res[i] = out
+	}
+
+	return res, nil
+}
+
+func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) {
+	var out struct {
+		Addrs map[string][]string
+	}
+	if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+	res := map[peer.ID][]multiaddr.Multiaddr{}
+	for spid, saddrs := range out.Addrs {
+		addrs := make([]multiaddr.Multiaddr, len(saddrs))
+
+		for i, addr := range saddrs {
+			a, err := multiaddr.NewMultiaddr(addr)
+			if err != nil {
+				return nil, err
+			}
+			addrs[i] = a
+		}
+
+		pid, err := peer.Decode(spid)
+		if err != nil {
+			return nil, err
+		}
+
+		res[pid] = addrs
+	}
+
+	return res, nil
+}
+
+func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]multiaddr.Multiaddr, len(out.Strings))
+	for i, addr := range out.Strings {
+		ma, err := multiaddr.NewMultiaddr(addr)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = ma
+	}
+	return res, nil
+}
+
+func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) {
+	var out struct {
+		Strings []string
+	}
+
+	if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil {
+		return nil, err
+	}
+
+	res := make([]multiaddr.Multiaddr, len(out.Strings))
+	for i, addr := range out.Strings {
+		ma, err := multiaddr.NewMultiaddr(addr)
+		if err != nil {
+			return nil, err
+		}
+		res[i] = ma
+	}
+	return res, nil
+}
+
+func (api *SwarmAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

client/rpc/unixfs.go

@@ -0,0 +1,230 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+
+	iface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
+	"github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/boxo/files"
+	unixfs "github.com/ipfs/boxo/ipld/unixfs"
+	unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb"
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type addEvent struct {
+	Name  string
+	Hash  string `json:",omitempty"`
+	Bytes int64  `json:",omitempty"`
+	Size  string `json:",omitempty"`
+}
+
+type UnixfsAPI HttpApi
+
+func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.Resolved, error) {
+	options, _, err := caopts.UnixfsAddOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	mht, ok := mh.Codes[options.MhType]
+	if !ok {
+		return nil, fmt.Errorf("unknowm mhType %d", options.MhType)
+	}
+
+	req := api.core().Request("add").
+		Option("hash", mht).
+		Option("chunker", options.Chunker).
+		Option("cid-version", options.CidVersion).
+		Option("fscache", options.FsCache).
+		Option("inline", options.Inline).
+		Option("inline-limit", options.InlineLimit).
+		Option("nocopy", options.NoCopy).
+		Option("only-hash", options.OnlyHash).
+		Option("pin", options.Pin).
+		Option("silent", options.Silent).
+		Option("progress", options.Progress)
+
+	if options.RawLeavesSet {
+		req.Option("raw-leaves", options.RawLeaves)
+	}
+
+	switch options.Layout {
+	case caopts.BalancedLayout:
+		// noop, default
+	case caopts.TrickleLayout:
+		req.Option("trickle", true)
+	}
+
+	d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side
+	req.Body(files.NewMultiFileReader(d, false))
+
+	var out addEvent
+	resp, err := req.Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	defer resp.Output.Close()
+	dec := json.NewDecoder(resp.Output)
+loop:
+	for {
+		var evt addEvent
+		switch err := dec.Decode(&evt); err {
+		case nil:
+		case io.EOF:
+			break loop
+		default:
+			return nil, err
+		}
+		out = evt
+
+		if options.Events != nil {
+			ifevt := &iface.AddEvent{
+				Name:  out.Name,
+				Size:  out.Size,
+				Bytes: out.Bytes,
+			}
+
+			if out.Hash != "" {
+				c, err := cid.Parse(out.Hash)
+				if err != nil {
+					return nil, err
+				}
+
+				ifevt.Path = path.IpfsPath(c)
+			}
+
+			select {
+			case options.Events <- ifevt:
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		}
+	}
+
+	c, err := cid.Parse(out.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return path.IpfsPath(c), nil
+}
+
+type lsLink struct {
+	Name, Hash string
+	Size       uint64
+	Type       unixfs_pb.Data_DataType
+	Target     string
+}
+
+type lsObject struct {
+	Hash  string
+	Links []lsLink
+}
+
+type lsOutput struct {
+	Objects []lsObject
+}
+
+func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) {
+	options, err := caopts.UnixfsLsOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := api.core().Request("ls", p.String()).
+		Option("resolve-type", options.ResolveChildren).
+		Option("size", options.ResolveChildren).
+		Option("stream", true).
+		Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	dec := json.NewDecoder(resp.Output)
+	out := make(chan iface.DirEntry)
+
+	go func() {
+		defer resp.Close()
+		defer close(out)
+
+		for {
+			var link lsOutput
+			if err := dec.Decode(&link); err != nil {
+				if err == io.EOF {
+					return
+				}
+				select {
+				case out <- iface.DirEntry{Err: err}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			if len(link.Objects) != 1 {
+				select {
+				case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			if len(link.Objects[0].Links) != 1 {
+				select {
+				case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			l0 := link.Objects[0].Links[0]
+
+			c, err := cid.Decode(l0.Hash)
+			if err != nil {
+				select {
+				case out <- iface.DirEntry{Err: err}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			var ftype iface.FileType
+			switch l0.Type {
+			case unixfs.TRaw, unixfs.TFile:
+				ftype = iface.TFile
+			case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata:
+				ftype = iface.TDirectory
+			case unixfs.TSymlink:
+				ftype = iface.TSymlink
+			}
+
+			select {
+			case out <- iface.DirEntry{
+				Name:   l0.Name,
+				Cid:    c,
+				Size:   l0.Size,
+				Type:   ftype,
+				Target: l0.Target,
+			}:
+			case <-ctx.Done():
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (api *UnixfsAPI) core() *HttpApi {
+	return (*HttpApi)(api)
+}

cmd/ipfs/daemon.go

@@ -425,11 +425,14 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	case routingOptionNoneKwd:
 		ncfg.Routing = libp2p.NilRouterOption
 	case routingOptionCustomKwd:
+		if cfg.Routing.AcceleratedDHTClient {
+			return fmt.Errorf("Routing.AcceleratedDHTClient option is set even tho Routing.Type is custom, using custom .AcceleratedDHTClient needs to be set on DHT routers individually")
+		}
 		ncfg.Routing = libp2p.ConstructDelegatedRouting(
 			cfg.Routing.Routers,
 			cfg.Routing.Methods,
 			cfg.Identity.PeerID,
-			cfg.Addresses.Swarm,
+			cfg.Addresses,
 			cfg.Identity.PrivKey,
 		)
 	default:

config/experiments.go

@@ -8,7 +8,7 @@ type Experiments struct {
 	Libp2pStreamMounting          bool
 	P2pHttpProxy                  bool //nolint
 	StrategicProviding            bool
-	AcceleratedDHTClient          bool
+	AcceleratedDHTClient          experimentalAcceleratedDHTClient `json:",omitempty"`
 	OptimisticProvide             bool
 	OptimisticProvideJobsPoolSize int
 }

config/gateway.go

@@ -1,6 +1,9 @@
 package config
 
-const DefaultInlineDNSLink = false
+const (
+	DefaultInlineDNSLink         = false
+	DefaultDeserializedResponses = true
+)
 
 type GatewaySpec struct {
 	// Paths is explicit list of path prefixes that should be handled by
@@ -25,6 +28,11 @@ type GatewaySpec struct {
 	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
 	// and Origin per CID isolation provided by rules like https://publicsuffix.org
 	InlineDNSLink Flag
+
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// responses. Disabling this option enables a Trustless Gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+	DeserializedResponses Flag
 }
 
 // Gateway contains options for the HTTP gateway server.
@@ -56,6 +64,12 @@ type Gateway struct {
 	// This flag can be overridden per FQDN in PublicGateways.
 	NoDNSLink bool
 
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// requests. Disabling this option enables a Trustless only gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can
+	// be overridden per FQDN in PublicGateways.
+	DeserializedResponses Flag
+
 	// PublicGateways configures behavior of known public gateways.
 	// Each key is a fully qualified domain name (FQDN).
 	PublicGateways map[string]*GatewaySpec

config/internal.go

@@ -2,9 +2,10 @@ package config
 
 type Internal struct {
 	// All marked as omitempty since we are expecting to make changes to all subcomponents of Internal
-	Bitswap                     *InternalBitswap `json:",omitempty"`
-	UnixFSShardingSizeThreshold *OptionalString  `json:",omitempty"`
-	Libp2pForceReachability     *OptionalString  `json:",omitempty"`
+	Bitswap                     *InternalBitswap  `json:",omitempty"`
+	UnixFSShardingSizeThreshold *OptionalString   `json:",omitempty"`
+	Libp2pForceReachability     *OptionalString   `json:",omitempty"`
+	BackupBootstrapInterval     *OptionalDuration `json:",omitempty"`
 }
 
 type InternalBitswap struct {

config/routing.go

@@ -15,6 +15,8 @@ type Routing struct {
 	// When "custom" is set, user-provided Routing.Routers is used.
 	Type *OptionalString `json:",omitempty"`
 
+	AcceleratedDHTClient bool
+
 	Routers Routers
 
 	Methods Methods

config/types.go

@@ -438,3 +438,27 @@ func (swarmLimits) UnmarshalJSON(b []byte) error {
 		}
 	}
 }
+
+type experimentalAcceleratedDHTClient struct{}
+
+var _ json.Unmarshaler = experimentalAcceleratedDHTClient{}
+
+func (experimentalAcceleratedDHTClient) UnmarshalJSON(b []byte) error {
+	d := json.NewDecoder(bytes.NewReader(b))
+	for {
+		switch tok, err := d.Token(); err {
+		case io.EOF:
+			return nil
+		case nil:
+			switch tok {
+			case json.Delim('{'), json.Delim('}'):
+				// accept empty objects
+				continue
+			}
+			//nolint
+			return fmt.Errorf("The Experimental.AcceleratedDHTClient key has been moved to Routing.AcceleratedDHTClient in Kubo 0.21, please use this new key and remove the old one.")
+		default:
+			return err
+		}
+	}
+}

core/bootstrap/bootstrap.go

@@ -3,16 +3,16 @@ package bootstrap
 import (
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	logging "github.com/ipfs/go-log"
 	"github.com/jbenet/goprocess"
-	"github.com/jbenet/goprocess/context"
-	"github.com/jbenet/goprocess/periodic"
+	goprocessctx "github.com/jbenet/goprocess/context"
+	periodicproc "github.com/jbenet/goprocess/periodic"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -50,13 +50,26 @@ type BootstrapConfig struct {
 	// for the bootstrap process to use. This makes it possible for clients
 	// to control the peers the process uses at any moment.
 	BootstrapPeers func() []peer.AddrInfo
+
+	// BackupBootstrapInterval governs the periodic interval at which the node will
+	// attempt to save connected nodes to use as temporary bootstrap peers.
+	BackupBootstrapInterval time.Duration
+
+	// MaxBackupBootstrapSize controls the maximum number of peers we're saving
+	// as backup bootstrap peers.
+	MaxBackupBootstrapSize int
+
+	SaveBackupBootstrapPeers func(context.Context, []peer.AddrInfo)
+	LoadBackupBootstrapPeers func(context.Context) []peer.AddrInfo
 }
 
 // DefaultBootstrapConfig specifies default sane parameters for bootstrapping.
 var DefaultBootstrapConfig = BootstrapConfig{
-	MinPeerThreshold:  4,
-	Period:            30 * time.Second,
-	ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3
+	MinPeerThreshold:        4,
+	Period:                  30 * time.Second,
+	ConnectionTimeout:       (30 * time.Second) / 3, // Perod / 3
+	BackupBootstrapInterval: 1 * time.Hour,
+	MaxBackupBootstrapSize:  20,
 }
 
 func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig {
@@ -90,6 +103,9 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf
 			log.Debugf("%s bootstrap error: %s", id, err)
 		}
 
+		// Exit the first call (triggered independently by `proc.Go`, not `Tick`)
+		// only after being done with the *single* Routing.Bootstrap call. Following
+		// periodic calls (`Tick`) will not block on this.
 		<-doneWithRound
 	}
 
@@ -108,9 +124,100 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf
 
 	doneWithRound <- struct{}{}
 	close(doneWithRound) // it no longer blocks periodic
+
+	startSavePeersAsTemporaryBootstrapProc(cfg, host, proc)
+
 	return proc, nil
 }
 
+// Aside of the main bootstrap process we also run a secondary one that saves
+// connected peers as a backup measure if we can't connect to the official
+// bootstrap ones. These peers will serve as *temporary* bootstrap nodes.
+func startSavePeersAsTemporaryBootstrapProc(cfg BootstrapConfig, host host.Host, bootstrapProc goprocess.Process) {
+	savePeersFn := func(worker goprocess.Process) {
+		ctx := goprocessctx.OnClosingContext(worker)
+
+		if err := saveConnectedPeersAsTemporaryBootstrap(ctx, host, cfg); err != nil {
+			log.Debugf("saveConnectedPeersAsTemporaryBootstrap error: %s", err)
+		}
+	}
+	savePeersProc := periodicproc.Tick(cfg.BackupBootstrapInterval, savePeersFn)
+
+	// When the main bootstrap process ends also terminate the 'save connected
+	// peers' ones. Coupling the two seems the easiest way to handle this backup
+	// process without additional complexity.
+	go func() {
+		<-bootstrapProc.Closing()
+		savePeersProc.Close()
+	}()
+
+	// Run the first round now (after the first bootstrap process has finished)
+	// as the SavePeersPeriod can be much longer than bootstrap.
+	savePeersProc.Go(savePeersFn)
+}
+
+func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, cfg BootstrapConfig) error {
+	// Randomize the list of connected peers, we don't prioritize anyone.
+	connectedPeers := randomizeList(host.Network().Peers())
+
+	bootstrapPeers := cfg.BootstrapPeers()
+	backupPeers := make([]peer.AddrInfo, 0, cfg.MaxBackupBootstrapSize)
+
+	// Choose peers to save and filter out the ones that are already bootstrap nodes.
+	for _, p := range connectedPeers {
+		found := false
+		for _, bootstrapPeer := range bootstrapPeers {
+			if p == bootstrapPeer.ID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			backupPeers = append(backupPeers, peer.AddrInfo{
+				ID:    p,
+				Addrs: host.Network().Peerstore().Addrs(p),
+			})
+		}
+
+		if len(backupPeers) >= cfg.MaxBackupBootstrapSize {
+			break
+		}
+	}
+
+	// If we didn't reach the target number use previously stored connected peers.
+	if len(backupPeers) < cfg.MaxBackupBootstrapSize {
+		oldSavedPeers := cfg.LoadBackupBootstrapPeers(ctx)
+		log.Debugf("missing %d peers to reach backup bootstrap target of %d, trying from previous list of %d saved peers",
+			cfg.MaxBackupBootstrapSize-len(backupPeers), cfg.MaxBackupBootstrapSize, len(oldSavedPeers))
+
+		// Add some of the old saved peers. Ensure we don't duplicate them.
+		for _, p := range oldSavedPeers {
+			found := false
+			for _, sp := range backupPeers {
+				if p.ID == sp.ID {
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				backupPeers = append(backupPeers, p)
+			}
+
+			if len(backupPeers) >= cfg.MaxBackupBootstrapSize {
+				break
+			}
+		}
+	}
+
+	cfg.SaveBackupBootstrapPeers(ctx, backupPeers)
+	log.Debugf("saved %d peers (of %d target) as bootstrap backup in the config", len(backupPeers), cfg.MaxBackupBootstrapSize)
+	return nil
+}
+
+// Connect to as many peers needed to reach the BootstrapConfig.MinPeerThreshold.
+// Peers can be original bootstrap or temporary ones (drawn from a list of
+// persisted previously connected peers).
 func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) error {
 
 	ctx, cancel := context.WithTimeout(ctx, cfg.ConnectionTimeout)
@@ -127,35 +234,58 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er
 			id, len(connected), cfg.MinPeerThreshold)
 		return nil
 	}
-	numToDial := cfg.MinPeerThreshold - len(connected)
+	numToDial := cfg.MinPeerThreshold - len(connected) // numToDial > 0
 
-	// filter out bootstrap nodes we are already connected to
-	var notConnected []peer.AddrInfo
-	for _, p := range peers {
-		if host.Network().Connectedness(p.ID) != network.Connected {
-			notConnected = append(notConnected, p)
+	if len(peers) > 0 {
+		numToDial -= int(peersConnect(ctx, host, peers, numToDial, true))
+		if numToDial <= 0 {
+			return nil
 		}
 	}
 
-	// if connected to all bootstrap peer candidates, exit
-	if len(notConnected) < 1 {
-		log.Debugf("%s no more bootstrap peers to create %d connections", id, numToDial)
-		return ErrNotEnoughBootstrapPeers
+	log.Debugf("not enough bootstrap peers to fill the remaining target of %d connections, trying backup list", numToDial)
+
+	tempBootstrapPeers := cfg.LoadBackupBootstrapPeers(ctx)
+	if len(tempBootstrapPeers) > 0 {
+		numToDial -= int(peersConnect(ctx, host, tempBootstrapPeers, numToDial, false))
+		if numToDial <= 0 {
+			return nil
+		}
 	}
 
-	// connect to a random susbset of bootstrap candidates
-	randSubset := randomSubsetOfPeers(notConnected, numToDial)
+	log.Debugf("tried both original bootstrap peers and temporary ones but still missing target of %d connections", numToDial)
 
-	log.Debugf("%s bootstrapping to %d nodes: %s", id, numToDial, randSubset)
-	return bootstrapConnect(ctx, host, randSubset)
+	return ErrNotEnoughBootstrapPeers
 }
 
-func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) error {
-	if len(peers) < 1 {
-		return ErrNotEnoughBootstrapPeers
-	}
+// Attempt to make `needed` connections from the `availablePeers` list. Mark
+// peers as either `permanent` or temporary when adding them to the Peerstore.
+// Return the number of connections completed. We eagerly over-connect in parallel,
+// so we might connect to more than needed.
+// (We spawn as many routines and attempt connections as the number of availablePeers,
+// but this list comes from restricted sets of original or temporary bootstrap
+// nodes which will keep it under a sane value.)
+func peersConnect(ctx context.Context, ph host.Host, availablePeers []peer.AddrInfo, needed int, permanent bool) uint64 {
+	peers := randomizeList(availablePeers)
+
+	// Monitor the number of connections and stop if we reach the target.
+	var connected uint64
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(1 * time.Second):
+				if int(atomic.LoadUint64(&connected)) >= needed {
+					cancel()
+					return
+				}
+			}
+		}
+	}()
 
-	errs := make(chan error, len(peers))
 	var wg sync.WaitGroup
 	for _, p := range peers {
 
@@ -164,45 +294,46 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo)
 		// fail/abort due to an expiring context.
 		// Also, performed asynchronously for dial speed.
 
+		if int(atomic.LoadUint64(&connected)) >= needed {
+			cancel()
+			break
+		}
+
 		wg.Add(1)
 		go func(p peer.AddrInfo) {
 			defer wg.Done()
-			log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID)
 
-			ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL)
-			if err := ph.Connect(ctx, p); err != nil {
-				log.Debugf("failed to bootstrap with %v: %s", p.ID, err)
-				errs <- err
+			// Skip addresses belonging to a peer we're already connected to.
+			// (Not a guarantee but a best-effort policy.)
+			if ph.Network().Connectedness(p.ID) == network.Connected {
 				return
 			}
+			log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID)
+
+			if err := ph.Connect(ctx, p); err != nil {
+				if ctx.Err() != context.Canceled {
+					log.Debugf("failed to bootstrap with %v: %s", p.ID, err)
+				}
+				return
+			}
+			if permanent {
+				// We're connecting to an original bootstrap peer, mark it as
+				// a permanent address (Connect will register it as TempAddrTTL).
+				ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL)
+			}
+
 			log.Infof("bootstrapped with %v", p.ID)
+			atomic.AddUint64(&connected, 1)
 		}(p)
 	}
 	wg.Wait()
 
-	// our failure condition is when no connection attempt succeeded.
-	// So drain the errs channel, counting the results.
-	close(errs)
-	count := 0
-	var err error
-	for err = range errs {
-		if err != nil {
-			count++
-		}
-	}
-	if count == len(peers) {
-		return fmt.Errorf("failed to bootstrap. %s", err)
-	}
-	return nil
+	return connected
 }
 
-func randomSubsetOfPeers(in []peer.AddrInfo, max int) []peer.AddrInfo {
-	if max > len(in) {
-		max = len(in)
-	}
-
-	out := make([]peer.AddrInfo, max)
-	for i, val := range rand.Perm(len(in))[:max] {
+func randomizeList[T any](in []T) []T {
+	out := make([]T, len(in))
+	for i, val := range rand.Perm(len(in)) {
 		out[i] = in[val]
 	}
 	return out

core/bootstrap/bootstrap_test.go

@@ -7,9 +7,9 @@ import (
 	"github.com/libp2p/go-libp2p/core/test"
 )
 
-func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) {
+func TestRandomizeAddressList(t *testing.T) {
 	var ps []peer.AddrInfo
-	sizeofSlice := 100
+	sizeofSlice := 10
 	for i := 0; i < sizeofSlice; i++ {
 		pid, err := test.RandPeerID()
 		if err != nil {
@@ -18,7 +18,7 @@ func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) {
 
 		ps = append(ps, peer.AddrInfo{ID: pid})
 	}
-	out := randomSubsetOfPeers(ps, 2*sizeofSlice)
+	out := randomizeList(ps)
 	if len(out) != len(ps) {
 		t.Fail()
 	}

core/commands/dag/dag.go

@@ -1,6 +1,8 @@
 package dagcmd
 
 import (
+	"encoding/csv"
+	"encoding/json"
 	"fmt"
 	"io"
 
@@ -276,12 +278,81 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/
 
 // DagStat is a dag stat command response
 type DagStat struct {
-	Size      uint64
-	NumBlocks int64
+	Cid       cid.Cid `json:",omitempty"`
+	Size      uint64  `json:",omitempty"`
+	NumBlocks int64   `json:",omitempty"`
 }
 
 func (s *DagStat) String() string {
-	return fmt.Sprintf("Size: %d, NumBlocks: %d", s.Size, s.NumBlocks)
+	return fmt.Sprintf("%s  %d  %d", s.Cid.String()[:20], s.Size, s.NumBlocks)
+}
+
+func (s *DagStat) MarshalJSON() ([]byte, error) {
+	type Alias DagStat
+	/*
+		We can't rely on cid.Cid.MarshalJSON since it uses the {"/": "..."}
+		format. To make the output consistent and follow the Kubo API patterns
+		we use the Cid.String method
+	*/
+	return json.Marshal(struct {
+		Cid string `json:"Cid"`
+		*Alias
+	}{
+		Cid:   s.Cid.String(),
+		Alias: (*Alias)(s),
+	})
+}
+
+func (s *DagStat) UnmarshalJSON(data []byte) error {
+	/*
+		We can't rely on cid.Cid.UnmarshalJSON since it uses the {"/": "..."}
+		format. To make the output consistent and follow the Kubo API patterns
+		we use the Cid.Parse method
+	*/
+	type Alias DagStat
+	aux := struct {
+		Cid string `json:"Cid"`
+		*Alias
+	}{
+		Alias: (*Alias)(s),
+	}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	Cid, err := cid.Parse(aux.Cid)
+	if err != nil {
+		return err
+	}
+	s.Cid = Cid
+	return nil
+}
+
+type DagStatSummary struct {
+	redundantSize uint64     `json:"-"`
+	UniqueBlocks  int        `json:",omitempty"`
+	TotalSize     uint64     `json:",omitempty"`
+	SharedSize    uint64     `json:",omitempty"`
+	Ratio         float32    `json:",omitempty"`
+	DagStatsArray []*DagStat `json:"DagStats,omitempty"`
+}
+
+func (s *DagStatSummary) String() string {
+	return fmt.Sprintf("Total Size: %d\nUnique Blocks: %d\nShared Size: %d\nRatio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio)
+}
+
+func (s *DagStatSummary) incrementTotalSize(size uint64) {
+	s.TotalSize += size
+}
+func (s *DagStatSummary) incrementRedundantSize(size uint64) {
+	s.redundantSize += size
+}
+func (s *DagStatSummary) appendStats(stats *DagStat) {
+	s.DagStatsArray = append(s.DagStatsArray, stats)
+}
+
+func (s *DagStatSummary) calculateSummary() {
+	s.Ratio = float32(s.redundantSize) / float32(s.TotalSize)
+	s.SharedSize = s.redundantSize - s.TotalSize
 }
 
 // DagStatCmd is a command for getting size information about an ipfs-stored dag
@@ -296,24 +367,50 @@ Note: This command skips duplicate blocks in reporting both size and the number
 `,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("root", true, false, "CID of a DAG root to get statistics for").EnableStdin(),
+		cmds.StringArg("root", true, true, "CID of a DAG root to get statistics for").EnableStdin(),
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption(progressOptionName, "p", "Return progressive data while reading through the DAG").WithDefault(true),
 	},
 	Run:  dagStat,
-	Type: DagStat{},
+	Type: DagStatSummary{},
 	PostRun: cmds.PostRunMap{
 		cmds.CLI: finishCLIStat,
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStat) error {
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error {
+			fmt.Fprintln(w)
+			csvWriter := csv.NewWriter(w)
+			csvWriter.Comma = '\t'
+			cidSpacing := len(event.DagStatsArray[0].Cid.String())
+			header := []string{fmt.Sprintf("%-*s", cidSpacing, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"}
+			if err := csvWriter.Write(header); err != nil {
+				return err
+			}
+			for _, dagStat := range event.DagStatsArray {
+				numBlocksStr := fmt.Sprint(dagStat.NumBlocks)
+				err := csvWriter.Write([]string{
+					dagStat.Cid.String(),
+					fmt.Sprintf("%-15s", numBlocksStr),
+					fmt.Sprint(dagStat.Size),
+				})
+				if err != nil {
+					return err
+				}
+			}
+			csvWriter.Flush()
+			fmt.Fprint(w, "\nSummary\n")
 			_, err := fmt.Fprintf(
 				w,
 				"%v\n",
 				event,
 			)
+			fmt.Fprint(w, "\n\n")
 			return err
 		}),
+		cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error {
+			return json.NewEncoder(w).Encode(event)
+		},
+		),
 	},
 }

core/commands/dag/import.go

@@ -27,6 +27,8 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 		return err
 	}
 
+	blockDecoder := ipldlegacy.NewDecoder()
+
 	// on import ensure we do not reach out to the network for any reason
 	// if a pin based on what is imported + what is in the blockstore
 	// isn't possible: tough luck
@@ -94,7 +96,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 				}
 
 				// the double-decode is suboptimal, but we need it for batching
-				nd, err := ipldlegacy.DecodeNode(req.Context, block)
+				nd, err := blockDecoder.DecodeNode(req.Context, block)
 				if err != nil {
 					return err
 				}
@@ -131,7 +133,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 			// and ensure the gray bucket is empty at the end (or use the network to download missing blocks).
 			if block, err := node.Blockstore.Get(req.Context, c); err != nil {
 				ret.PinErrorMsg = err.Error()
-			} else if nd, err := ipldlegacy.DecodeNode(req.Context, block); err != nil {
+			} else if nd, err := blockDecoder.DecodeNode(req.Context, block); err != nil {
 				ret.PinErrorMsg = err.Error()
 			} else if err := node.Pinning.Pin(req.Context, nd, true); err != nil {
 				ret.PinErrorMsg = err.Error()

core/commands/dag/stat.go

@@ -6,70 +6,81 @@ import (
 	"os"
 
 	"github.com/ipfs/boxo/coreiface/path"
+	mdag "github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/boxo/ipld/merkledag/traverse"
+	cid "github.com/ipfs/go-cid"
+	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 	"github.com/ipfs/kubo/core/commands/e"
-
-	mdag "github.com/ipfs/boxo/ipld/merkledag"
-	cmds "github.com/ipfs/go-ipfs-cmds"
 )
 
+// TODO cache every cid traversal in a dp cache
+// if the cid exists in the cache, don't traverse it, and use the cached result
+// to compute the new state
+
 func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 	progressive := req.Options[progressOptionName].(bool)
-
 	api, err := cmdenv.GetApi(env, req)
 	if err != nil {
 		return err
 	}
-
-	rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0]))
-	if err != nil {
-		return err
-	}
-
-	if len(rp.Remainder()) > 0 {
-		return fmt.Errorf("cannot return size for anything other than a DAG with a root CID")
-	}
-
 	nodeGetter := mdag.NewSession(req.Context, api.Dag())
-	obj, err := nodeGetter.Get(req.Context, rp.Cid())
-	if err != nil {
-		return err
-	}
 
-	dagstats := &DagStat{}
-	err = traverse.Traverse(obj, traverse.Options{
-		DAG:   nodeGetter,
-		Order: traverse.DFSPre,
-		Func: func(current traverse.State) error {
-			dagstats.Size += uint64(len(current.Node.RawData()))
-			dagstats.NumBlocks++
-
-			if progressive {
-				if err := res.Emit(dagstats); err != nil {
-					return err
-				}
-			}
-			return nil
-		},
-		ErrFunc:        nil,
-		SkipDuplicates: true,
-	})
-	if err != nil {
-		return fmt.Errorf("error traversing DAG: %w", err)
-	}
-
-	if !progressive {
-		if err := res.Emit(dagstats); err != nil {
+	cidSet := cid.NewSet()
+	dagStatSummary := &DagStatSummary{DagStatsArray: []*DagStat{}}
+	for _, a := range req.Arguments {
+		rp, err := api.ResolvePath(req.Context, path.New(a))
+		if err != nil {
 			return err
 		}
+		if len(rp.Remainder()) > 0 {
+			return fmt.Errorf("cannot return size for anything other than a DAG with a root CID")
+		}
+
+		obj, err := nodeGetter.Get(req.Context, rp.Cid())
+		if err != nil {
+			return err
+		}
+		dagstats := &DagStat{Cid: rp.Cid()}
+		dagStatSummary.appendStats(dagstats)
+		err = traverse.Traverse(obj, traverse.Options{
+			DAG:   nodeGetter,
+			Order: traverse.DFSPre,
+			Func: func(current traverse.State) error {
+				currentNodeSize := uint64(len(current.Node.RawData()))
+				dagstats.Size += currentNodeSize
+				dagstats.NumBlocks++
+				if !cidSet.Has(current.Node.Cid()) {
+					dagStatSummary.incrementTotalSize(currentNodeSize)
+				}
+				dagStatSummary.incrementRedundantSize(currentNodeSize)
+				cidSet.Add(current.Node.Cid())
+				if progressive {
+					if err := res.Emit(dagStatSummary); err != nil {
+						return err
+					}
+				}
+				return nil
+			},
+			ErrFunc:        nil,
+			SkipDuplicates: true,
+		})
+		if err != nil {
+			return fmt.Errorf("error traversing DAG: %w", err)
+		}
 	}
 
+	dagStatSummary.UniqueBlocks = cidSet.Len()
+	dagStatSummary.calculateSummary()
+
+	if err := res.Emit(dagStatSummary); err != nil {
+		return err
+	}
 	return nil
 }
 
 func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error {
-	var dagStats *DagStat
+	var dagStats *DagStatSummary
 	for {
 		v, err := res.Next()
 		if err != nil {
@@ -78,13 +89,20 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error {
 			}
 			return err
 		}
-
-		out, ok := v.(*DagStat)
-		if !ok {
+		switch out := v.(type) {
+		case *DagStatSummary:
+			dagStats = out
+			if dagStats.Ratio == 0 {
+				length := len(dagStats.DagStatsArray)
+				if length > 0 {
+					currentStat := dagStats.DagStatsArray[length-1]
+					fmt.Fprintf(os.Stderr, "CID: %s, Size: %d, NumBlocks: %d\n", currentStat.Cid, currentStat.Size, currentStat.NumBlocks)
+				}
+			}
+		default:
 			return e.TypeErr(out, v)
+
 		}
-		dagStats = out
-		fmt.Fprintf(os.Stderr, "%v\r", out)
 	}
 	return re.Emit(dagStats)
 }

core/commands/name/name.go

@@ -17,7 +17,6 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/dagcbor"
 	"github.com/ipld/go-ipld-prime/codec/dagjson"
-	ic "github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 	mbase "github.com/multiformats/go-multibase"
 )
@@ -216,33 +215,7 @@ Passing --verify will verify signature against provided public key.
 				PublicKey: id,
 			}
 
-			pub, err := id.ExtractPublicKey()
-			if err != nil {
-				// Make sure it works with all those RSA that cannot be embedded into the
-				// Peer ID.
-				if len(entry.PubKey) > 0 {
-					pub, err = ic.UnmarshalPublicKey(entry.PubKey)
-					if err != nil {
-						return err
-					}
-
-					// Verify the public key matches the name we are verifying.
-					entryID, err := peer.IDFromPublicKey(pub)
-
-					if err != nil {
-						return err
-					}
-
-					if id != entryID {
-						return fmt.Errorf("record public key does not match the verified name")
-					}
-				}
-			}
-			if err != nil {
-				return err
-			}
-
-			err = ipns.Validate(pub, &entry)
+			err = ipns.ValidateWithPeerID(id, &entry)
 			if err == nil {
 				result.Validation.Valid = true
 			} else {

core/commands/pin/pin.go

@@ -342,14 +342,17 @@ Example:
 		}
 
 		// For backward compatibility, we accumulate the pins in the same output type as before.
-		emit := res.Emit
+		var emit func(PinLsOutputWrapper) error
 		lgcList := map[string]PinLsType{}
 		if !stream {
-			emit = func(v interface{}) error {
-				obj := v.(*PinLsOutputWrapper)
-				lgcList[obj.PinLsObject.Cid] = PinLsType{Type: obj.PinLsObject.Type}
+			emit = func(v PinLsOutputWrapper) error {
+				lgcList[v.PinLsObject.Cid] = PinLsType{Type: v.PinLsObject.Type}
 				return nil
 			}
+		} else {
+			emit = func(v PinLsOutputWrapper) error {
+				return res.Emit(v)
+			}
 		}
 
 		if len(req.Arguments) > 0 {
@@ -362,16 +365,16 @@ Example:
 		}
 
 		if !stream {
-			return cmds.EmitOnce(res, &PinLsOutputWrapper{
+			return cmds.EmitOnce(res, PinLsOutputWrapper{
 				PinLsList: PinLsList{Keys: lgcList},
 			})
 		}
 
 		return nil
 	},
-	Type: &PinLsOutputWrapper{},
+	Type: PinLsOutputWrapper{},
 	Encoders: cmds.EncoderMap{
-		cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinLsOutputWrapper) error {
+		cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error {
 			stream, _ := req.Options[pinStreamOptionName].(bool)
 
 			enc := json.NewEncoder(w)
@@ -382,7 +385,7 @@ Example:
 
 			return enc.Encode(out.PinLsList)
 		}),
-		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinLsOutputWrapper) error {
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error {
 			quiet, _ := req.Options[pinQuietOptionName].(bool)
 			stream, _ := req.Options[pinStreamOptionName].(bool)
 
@@ -418,7 +421,7 @@ type PinLsOutputWrapper struct {
 
 // PinLsList is a set of pins with their type
 type PinLsList struct {
-	Keys map[string]PinLsType
+	Keys map[string]PinLsType `json:",omitempty"`
 }
 
 // PinLsType contains the type of a pin
@@ -432,7 +435,7 @@ type PinLsObject struct {
 	Type string `json:",omitempty"`
 }
 
-func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value interface{}) error) error {
+func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error {
 	enc, err := cmdenv.GetCidEncoder(req)
 	if err != nil {
 		return err
@@ -470,7 +473,7 @@ func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fu
 			pinType = "indirect through " + pinType
 		}
 
-		err = emit(&PinLsOutputWrapper{
+		err = emit(PinLsOutputWrapper{
 			PinLsObject: PinLsObject{
 				Type: pinType,
 				Cid:  enc.Encode(rp.Cid()),
@@ -484,7 +487,7 @@ func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fu
 	return nil
 }
 
-func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value interface{}) error) error {
+func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error {
 	enc, err := cmdenv.GetCidEncoder(req)
 	if err != nil {
 		return err
@@ -511,7 +514,7 @@ func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fun
 		if err := p.Err(); err != nil {
 			return err
 		}
-		err = emit(&PinLsOutputWrapper{
+		err = emit(PinLsOutputWrapper{
 			PinLsObject: PinLsObject{
 				Type: p.Type(),
 				Cid:  enc.Encode(p.Path().Cid()),
@@ -648,13 +651,14 @@ var verifyPinCmd = &cmds.Command{
 
 // PinVerifyRes is the result returned for each pin checked in "pin verify"
 type PinVerifyRes struct {
-	Cid string
+	Cid string `json:",omitempty"`
+	Err string `json:",omitempty"`
 	PinStatus
 }
 
 // PinStatus is part of PinVerifyRes, do not use directly
 type PinStatus struct {
-	Ok       bool
+	Ok       bool      `json:",omitempty"`
 	BadNodes []BadNode `json:",omitempty"`
 }
 
@@ -669,16 +673,13 @@ type pinVerifyOpts struct {
 	includeOk bool
 }
 
-func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) (<-chan interface{}, error) {
+// FIXME: this implementation is duplicated sith core/coreapi.PinAPI.Verify, remove this one and exclusively rely on CoreAPI.
+func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) (<-chan any, error) {
 	visited := make(map[cid.Cid]PinStatus)
 
 	bs := n.Blocks.Blockstore()
 	DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
 	getLinks := dag.GetLinksWithDAG(DAG)
-	recPins, err := n.Pinning.RecursiveKeys(ctx)
-	if err != nil {
-		return nil, err
-	}
 
 	var checkPin func(root cid.Cid) PinStatus
 	checkPin = func(root cid.Cid) PinStatus {
@@ -719,14 +720,18 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci
 		return status
 	}
 
-	out := make(chan interface{})
+	out := make(chan any)
 	go func() {
 		defer close(out)
-		for _, cid := range recPins {
-			pinStatus := checkPin(cid)
+		for p := range n.Pinning.RecursiveKeys(ctx) {
+			if p.Err != nil {
+				out <- PinVerifyRes{Err: p.Err.Error()}
+				return
+			}
+			pinStatus := checkPin(p.C)
 			if !pinStatus.Ok || opts.includeOk {
 				select {
-				case out <- &PinVerifyRes{enc.Encode(cid), pinStatus}:
+				case out <- PinVerifyRes{Cid: enc.Encode(p.C), PinStatus: pinStatus}:
 				case <-ctx.Done():
 					return
 				}
@@ -739,12 +744,18 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci
 
 // Format formats PinVerifyRes
 func (r PinVerifyRes) Format(out io.Writer) {
+	if r.Err != "" {
+		fmt.Fprintf(out, "error: %s\n", r.Err)
+		return
+	}
+
 	if r.Ok {
 		fmt.Fprintf(out, "%s ok\n", r.Cid)
-	} else {
-		fmt.Fprintf(out, "%s broken\n", r.Cid)
-		for _, e := range r.BadNodes {
-			fmt.Fprintf(out, "  %s: %s\n", e.Cid, e.Err)
-		}
+		return
+	}
+
+	fmt.Fprintf(out, "%s broken\n", r.Cid)
+	for _, e := range r.BadNodes {
+		fmt.Fprintf(out, "  %s: %s\n", e.Cid, e.Err)
 	}
 }

core/commands/root.go

@@ -17,6 +17,7 @@ import (
 var log = logging.Logger("core/commands")
 
 var ErrNotOnline = errors.New("this command must be run in online mode. Try running 'ipfs daemon' first")
+var ErrSelfUnsupported = errors.New("finding your own node in the DHT is currently not supported")
 
 const (
 	RepoDirOption    = "repo-dir"

core/commands/routing.go

@@ -10,6 +10,8 @@ import (
 
 	cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
 
+	iface "github.com/ipfs/boxo/coreiface"
+	"github.com/ipfs/boxo/coreiface/options"
 	dag "github.com/ipfs/boxo/ipld/merkledag"
 	path "github.com/ipfs/boxo/path"
 	cid "github.com/ipfs/go-cid"
@@ -19,6 +21,16 @@ import (
 	routing "github.com/libp2p/go-libp2p/core/routing"
 )
 
+var (
+	errAllowOffline = errors.New("can't put while offline: pass `--allow-offline` to override")
+)
+
+const (
+	dhtVerboseOptionName   = "verbose"
+	numProvidersOptionName = "num-providers"
+	allowOfflineOptionName = "allow-offline"
+)
+
 var RoutingCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline:          "Issue routing commands.",
@@ -34,14 +46,6 @@ var RoutingCmd = &cmds.Command{
 	},
 }
 
-const (
-	dhtVerboseOptionName = "verbose"
-)
-
-const (
-	numProvidersOptionName = "num-providers"
-)
-
 var findProvidersRoutingCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline:          "Find peers that can provide a specific value, given a key.",
@@ -297,6 +301,10 @@ var findPeerRoutingCmd = &cmds.Command{
 			return err
 		}
 
+		if pid == nd.Identity {
+			return ErrSelfUnsupported
+		}
+
 		ctx, cancel := context.WithCancel(req.Context)
 		ctx, events := routing.RegisterForQueryEvents(ctx)
 
@@ -420,6 +428,9 @@ identified by QmFoo.
 		cmds.StringArg("key", true, false, "The key to store the value at."),
 		cmds.FileArg("value-file", true, false, "A path to a file containing the value to store.").EnableStdin(),
 	},
+	Options: []cmds.Option{
+		cmds.BoolOption(allowOfflineOptionName, "When offline, save the IPNS record to the the local datastore without broadcasting to the network instead of simply failing."),
+	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
@@ -437,13 +448,22 @@ identified by QmFoo.
 			return err
 		}
 
-		err = api.Routing().Put(req.Context, req.Arguments[0], data)
+		allowOffline, _ := req.Options[allowOfflineOptionName].(bool)
+
+		opts := []options.RoutingPutOption{
+			options.Put.AllowOffline(allowOffline),
+		}
+
+		err = api.Routing().Put(req.Context, req.Arguments[0], data, opts...)
 		if err != nil {
 			return err
 		}
 
 		id, err := api.Key().Self(req.Context)
 		if err != nil {
+			if err == iface.ErrOffline {
+				err = errAllowOffline
+			}
 			return err
 		}
 

core/commands/stat_provide.go

@@ -7,10 +7,10 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/ipfs/boxo/provider"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
-
-	"github.com/ipfs/boxo/provider/batched"
+	"golang.org/x/exp/constraints"
 )
 
 var statProvideCmd = &cmds.Command{
@@ -34,12 +34,7 @@ This interface is not stable and may change from release to release.
 			return ErrNotOnline
 		}
 
-		sys, ok := nd.Provider.(*batched.BatchProvidingSystem)
-		if !ok {
-			return fmt.Errorf("can only return stats if Experimental.AcceleratedDHTClient is enabled")
-		}
-
-		stats, err := sys.Stat(req.Context)
+		stats, err := nd.Provider.Stat()
 		if err != nil {
 			return err
 		}
@@ -51,7 +46,7 @@ This interface is not stable and may change from release to release.
 		return nil
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *batched.BatchedProviderStats) error {
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *provider.ReproviderStats) error {
 			wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0)
 			defer wtr.Flush()
 
@@ -62,14 +57,14 @@ This interface is not stable and may change from release to release.
 			return nil
 		}),
 	},
-	Type: batched.BatchedProviderStats{},
+	Type: provider.ReproviderStats{},
 }
 
 func humanDuration(val time.Duration) string {
 	return val.Truncate(time.Microsecond).String()
 }
 
-func humanNumber(n int) string {
+func humanNumber[T constraints.Float | constraints.Integer](n T) string {
 	nf := float64(n)
 	str := humanSI(nf, 0)
 	fullStr := humanFull(nf, 0)

core/core.go

@@ -11,10 +11,13 @@ package core
 
 import (
 	"context"
+	"encoding/json"
 	"io"
+	"time"
 
 	"github.com/ipfs/boxo/filestore"
 	pin "github.com/ipfs/boxo/pinning/pinner"
+	"github.com/ipfs/go-datastore"
 
 	bserv "github.com/ipfs/boxo/blockservice"
 	bstore "github.com/ipfs/boxo/blockstore"
@@ -46,6 +49,7 @@ import (
 
 	"github.com/ipfs/boxo/namesys"
 	ipnsrp "github.com/ipfs/boxo/namesys/republisher"
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/bootstrap"
 	"github.com/ipfs/kubo/core/node"
 	"github.com/ipfs/kubo/core/node/libp2p"
@@ -165,12 +169,40 @@ func (n *IpfsNode) Bootstrap(cfg bootstrap.BootstrapConfig) error {
 			return ps
 		}
 	}
+	if cfg.SaveBackupBootstrapPeers == nil {
+		cfg.SaveBackupBootstrapPeers = func(ctx context.Context, peerList []peer.AddrInfo) {
+			err := n.saveTempBootstrapPeers(ctx, peerList)
+			if err != nil {
+				log.Warnf("saveTempBootstrapPeers failed: %s", err)
+				return
+			}
+		}
+	}
+	if cfg.LoadBackupBootstrapPeers == nil {
+		cfg.LoadBackupBootstrapPeers = func(ctx context.Context) []peer.AddrInfo {
+			peerList, err := n.loadTempBootstrapPeers(ctx)
+			if err != nil {
+				log.Warnf("loadTempBootstrapPeers failed: %s", err)
+				return nil
+			}
+			return peerList
+		}
+	}
+
+	repoConf, err := n.Repo.Config()
+	if err != nil {
+		return err
+	}
+	if repoConf.Internal.BackupBootstrapInterval != nil {
+		cfg.BackupBootstrapInterval = repoConf.Internal.BackupBootstrapInterval.WithDefault(time.Hour)
+	}
 
-	var err error
 	n.Bootstrapper, err = bootstrap.Bootstrap(n.Identity, n.PeerHost, n.Routing, cfg)
 	return err
 }
 
+var TempBootstrapPeersKey = datastore.NewKey("/local/temp_bootstrap_peers")
+
 func (n *IpfsNode) loadBootstrapPeers() ([]peer.AddrInfo, error) {
 	cfg, err := n.Repo.Config()
 	if err != nil {
@@ -180,6 +212,33 @@ func (n *IpfsNode) loadBootstrapPeers() ([]peer.AddrInfo, error) {
 	return cfg.BootstrapPeers()
 }
 
+func (n *IpfsNode) saveTempBootstrapPeers(ctx context.Context, peerList []peer.AddrInfo) error {
+	ds := n.Repo.Datastore()
+	bytes, err := json.Marshal(config.BootstrapPeerStrings(peerList))
+	if err != nil {
+		return err
+	}
+
+	if err := ds.Put(ctx, TempBootstrapPeersKey, bytes); err != nil {
+		return err
+	}
+	return ds.Sync(ctx, TempBootstrapPeersKey)
+}
+
+func (n *IpfsNode) loadTempBootstrapPeers(ctx context.Context) ([]peer.AddrInfo, error) {
+	ds := n.Repo.Datastore()
+	bytes, err := ds.Get(ctx, TempBootstrapPeersKey)
+	if err != nil {
+		return nil, err
+	}
+
+	var addrs []string
+	if err := json.Unmarshal(bytes, &addrs); err != nil {
+		return nil, err
+	}
+	return config.ParseBootstrapPeers(addrs)
+}
+
 type ConstructPeerHostOpts struct {
 	AddrsFactory      p2pbhost.AddrsFactory
 	DisableNatPortMap bool

core/core_test.go

@@ -256,7 +256,7 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode {
 				cfg.Routing.Routers,
 				cfg.Routing.Methods,
 				cfg.Identity.PeerID,
-				cfg.Addresses.Swarm,
+				cfg.Addresses,
 				cfg.Identity.PrivKey,
 			),
 		},

core/coreapi/coreapi.go

@@ -238,7 +238,7 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 			return nil, fmt.Errorf("error constructing namesys: %w", err)
 		}
 
-		subAPI.provider = provider.NewOfflineProvider()
+		subAPI.provider = provider.NewNoopProvider()
 
 		subAPI.peerstore = nil
 		subAPI.peerHost = nil

core/coreapi/pin.go

@@ -12,9 +12,10 @@ import (
 	"github.com/ipfs/boxo/ipld/merkledag"
 	pin "github.com/ipfs/boxo/pinning/pinner"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/kubo/tracing"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+
+	"github.com/ipfs/kubo/tracing"
 )
 
 type PinAPI CoreAPI
@@ -156,6 +157,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt
 }
 
 type pinStatus struct {
+	err      error
 	cid      cid.Cid
 	ok       bool
 	badNodes []coreiface.BadPinNode
@@ -175,6 +177,10 @@ func (s *pinStatus) BadNodes() []coreiface.BadPinNode {
 	return s.badNodes
 }
 
+func (s *pinStatus) Err() error {
+	return s.err
+}
+
 func (n *badNode) Path() path.Resolved {
 	return n.path
 }
@@ -191,10 +197,6 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro
 	bs := api.blockstore
 	DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
 	getLinks := merkledag.GetLinksWithDAG(DAG)
-	recPins, err := api.pinning.RecursiveKeys(ctx)
-	if err != nil {
-		return nil, err
-	}
 
 	var checkPin func(root cid.Cid) *pinStatus
 	checkPin = func(root cid.Cid) *pinStatus {
@@ -229,8 +231,18 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro
 	out := make(chan coreiface.PinStatus)
 	go func() {
 		defer close(out)
-		for _, c := range recPins {
-			out <- checkPin(c)
+		for p := range api.pinning.RecursiveKeys(ctx) {
+			var res *pinStatus
+			if p.Err != nil {
+				res = &pinStatus{err: p.Err}
+			} else {
+				res = checkPin(p.C)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case out <- res:
+			}
 		}
 	}()
 
@@ -262,98 +274,92 @@ func (p *pinInfo) Err() error {
 func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreiface.Pin {
 	out := make(chan coreiface.Pin, 1)
 
-	keys := cid.NewSet()
+	emittedSet := cid.NewSet()
 
-	AddToResultKeys := func(keyList []cid.Cid, typeStr string) error {
-		for _, c := range keyList {
-			if keys.Visit(c) {
-				select {
-				case out <- &pinInfo{
-					pinType: typeStr,
-					path:    path.IpldPath(c),
-				}:
-				case <-ctx.Done():
-					return ctx.Err()
-				}
+	AddToResultKeys := func(c cid.Cid, typeStr string) error {
+		if emittedSet.Visit(c) {
+			select {
+			case out <- &pinInfo{
+				pinType: typeStr,
+				path:    path.IpldPath(c),
+			}:
+			case <-ctx.Done():
+				return ctx.Err()
 			}
 		}
 		return nil
 	}
 
-	VisitKeys := func(keyList []cid.Cid) {
-		for _, c := range keyList {
-			keys.Visit(c)
-		}
-	}
-
 	go func() {
 		defer close(out)
 
-		var dkeys, rkeys []cid.Cid
+		var rkeys []cid.Cid
 		var err error
 		if typeStr == "recursive" || typeStr == "all" {
-			rkeys, err = api.pinning.RecursiveKeys(ctx)
-			if err != nil {
-				out <- &pinInfo{err: err}
-				return
-			}
-			if err = AddToResultKeys(rkeys, "recursive"); err != nil {
-				out <- &pinInfo{err: err}
-				return
-			}
-		}
-		if typeStr == "direct" || typeStr == "all" {
-			dkeys, err = api.pinning.DirectKeys(ctx)
-			if err != nil {
-				out <- &pinInfo{err: err}
-				return
-			}
-			if err = AddToResultKeys(dkeys, "direct"); err != nil {
-				out <- &pinInfo{err: err}
-				return
-			}
-		}
-		if typeStr == "all" {
-			set := cid.NewSet()
-			for _, k := range rkeys {
-				err = merkledag.Walk(
-					ctx, merkledag.GetLinksWithDAG(api.dag), k,
-					set.Visit,
-					merkledag.SkipRoot(), merkledag.Concurrent(),
-				)
-				if err != nil {
+			for streamedCid := range api.pinning.RecursiveKeys(ctx) {
+				if streamedCid.Err != nil {
+					out <- &pinInfo{err: streamedCid.Err}
+					return
+				}
+				if err = AddToResultKeys(streamedCid.C, "recursive"); err != nil {
 					out <- &pinInfo{err: err}
 					return
 				}
+				rkeys = append(rkeys, streamedCid.C)
 			}
-			if err = AddToResultKeys(set.Keys(), "indirect"); err != nil {
-				out <- &pinInfo{err: err}
-				return
+		}
+		if typeStr == "direct" || typeStr == "all" {
+			for streamedCid := range api.pinning.DirectKeys(ctx) {
+				if streamedCid.Err != nil {
+					out <- &pinInfo{err: streamedCid.Err}
+					return
+				}
+				if err = AddToResultKeys(streamedCid.C, "direct"); err != nil {
+					out <- &pinInfo{err: err}
+					return
+				}
 			}
 		}
 		if typeStr == "indirect" {
 			// We need to first visit the direct pins that have priority
 			// without emitting them
 
-			dkeys, err = api.pinning.DirectKeys(ctx)
-			if err != nil {
-				out <- &pinInfo{err: err}
-				return
+			for streamedCid := range api.pinning.DirectKeys(ctx) {
+				if streamedCid.Err != nil {
+					out <- &pinInfo{err: streamedCid.Err}
+					return
+				}
+				emittedSet.Add(streamedCid.C)
 			}
-			VisitKeys(dkeys)
 
-			rkeys, err = api.pinning.RecursiveKeys(ctx)
-			if err != nil {
-				out <- &pinInfo{err: err}
-				return
+			for streamedCid := range api.pinning.RecursiveKeys(ctx) {
+				if streamedCid.Err != nil {
+					out <- &pinInfo{err: streamedCid.Err}
+					return
+				}
+				emittedSet.Add(streamedCid.C)
+				rkeys = append(rkeys, streamedCid.C)
 			}
-			VisitKeys(rkeys)
-
-			set := cid.NewSet()
+		}
+		if typeStr == "indirect" || typeStr == "all" {
+			walkingSet := cid.NewSet()
 			for _, k := range rkeys {
 				err = merkledag.Walk(
 					ctx, merkledag.GetLinksWithDAG(api.dag), k,
-					set.Visit,
+					func(c cid.Cid) bool {
+						if !walkingSet.Visit(c) {
+							return false
+						}
+						if emittedSet.Has(c) {
+							return true // skipped
+						}
+						err := AddToResultKeys(c, "indirect")
+						if err != nil {
+							out <- &pinInfo{err: err}
+							return false
+						}
+						return true
+					},
 					merkledag.SkipRoot(), merkledag.Concurrent(),
 				)
 				if err != nil {
@@ -361,10 +367,6 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreifac
 					return
 				}
 			}
-			if err = AddToResultKeys(set.Keys(), "indirect"); err != nil {
-				out <- &pinInfo{err: err}
-				return
-			}
 		}
 	}()
 

core/coreapi/routing.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	coreiface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
 	"github.com/ipfs/boxo/path"
 	peer "github.com/libp2p/go-libp2p/core/peer"
 )
@@ -24,9 +25,15 @@ func (r *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.routing.GetValue(ctx, dhtKey)
 }
 
-func (r *RoutingAPI) Put(ctx context.Context, key string, value []byte) error {
-	if !r.nd.IsOnline {
-		return coreiface.ErrOffline
+func (r *RoutingAPI) Put(ctx context.Context, key string, value []byte, opts ...caopts.RoutingPutOption) error {
+	options, err := caopts.RoutingPutOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	err = r.checkOnline(options.AllowOffline)
+	if err != nil {
+		return err
 	}
 
 	dhtKey, err := normalizeKey(key)

core/coreapi/test/api_test.go

@@ -31,7 +31,7 @@ const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe"
 
 type NodeProvider struct{}
 
-func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) {
+func (NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity bool, online bool, n int) ([]coreiface.CoreAPI, error) {
 	mn := mocknet.New()
 
 	nodes := make([]*core.IpfsNode, n)
@@ -82,7 +82,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 			Routing: libp2p.DHTServerOption,
 			Repo:    r,
 			Host:    mock.MockHostOption(mn),
-			Online:  fullIdentity,
+			Online:  online,
 			ExtraOpts: map[string]bool{
 				"pubsub": true,
 			},
@@ -102,15 +102,17 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
-	bsinf := bootstrap.BootstrapConfigWithPeers(
-		[]peer.AddrInfo{
-			nodes[0].Peerstore.PeerInfo(nodes[0].Identity),
-		},
-	)
+	if online {
+		bsinf := bootstrap.BootstrapConfigWithPeers(
+			[]peer.AddrInfo{
+				nodes[0].Peerstore.PeerInfo(nodes[0].Identity),
+			},
+		)
 
-	for _, n := range nodes[1:] {
-		if err := n.Bootstrap(bsinf); err != nil {
-			return nil, err
+		for _, n := range nodes[1:] {
+			if err := n.Bootstrap(bsinf); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -118,5 +120,5 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 }
 
 func TestIface(t *testing.T) {
-	tests.TestApi(&NodeProvider{})(t)
+	tests.TestApi(NodeProvider{})(t)
 }

core/coreapi/test/path_test.go

@@ -19,7 +19,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
 	defer cancel()
 
 	// Create a node
-	apis, err := NodeProvider{}.MakeAPISwarm(ctx, true, 1)
+	apis, err := NodeProvider{}.MakeAPISwarm(t, ctx, true, true, 1)
 	if err != nil {
 		t.Fatal(err)
 	}

core/corehttp/gateway.go

@@ -28,35 +28,21 @@ import (
 
 func GatewayOption(paths ...string) ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		config, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
 
-		headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
-		for h, v := range cfg.Gateway.HTTPHeaders {
-			headers[http.CanonicalHeaderKey(h)] = v
-		}
-
-		gateway.AddAccessControlHeaders(headers)
-
-		gwConfig := gateway.Config{
-			Headers: headers,
-		}
-
-		gwAPI, err := newGatewayBackend(n)
+		backend, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
 		}
 
-		gw := gateway.NewHandler(gwConfig, gwAPI)
-		gw = otelhttp.NewHandler(gw, "Gateway")
-
-		// By default, our HTTP handler is the gateway handler.
-		handler := gw.ServeHTTP
+		handler := gateway.NewHandler(config, backend)
+		handler = otelhttp.NewHandler(handler, "Gateway")
 
 		for _, p := range paths {
-			mux.HandleFunc(p+"/", handler)
+			mux.HandleFunc(p+"/", handler.ServeHTTP)
 		}
 
 		return mux, nil
@@ -65,19 +51,18 @@ func GatewayOption(paths ...string) ServeOption {
 
 func HostnameOption() ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		config, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
 
-		gwAPI, err := newGatewayBackend(n)
+		backend, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
 		}
 
-		publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways)
 		childMux := http.NewServeMux()
-		mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP)
+		mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP)
 		return childMux, nil
 	}
 }
@@ -123,11 +108,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
 		}
 	}
 
-	gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
+	backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
 	if err != nil {
 		return nil, err
 	}
-	return &offlineGatewayErrWrapper{gwimpl: gw}, nil
+	return &offlineGatewayErrWrapper{gwimpl: backend}, nil
 }
 
 type offlineGatewayErrWrapper struct {
@@ -171,10 +156,10 @@ func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway
 	return md, err
 }
 
-func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
-	md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
+func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) {
+	md, data, err := o.gwimpl.GetCAR(ctx, path, params)
 	err = offlineErrWrap(err)
-	return md, data, errCh, err
+	return md, data, err
 }
 
 func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
@@ -203,39 +188,58 @@ var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
 
 var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
 
-var subdomainGatewaySpec = &gateway.Specification{
+var subdomainGatewaySpec = &gateway.PublicGateway{
 	Paths:         defaultPaths,
 	UseSubdomains: true,
 }
 
-var defaultKnownGateways = map[string]*gateway.Specification{
+var defaultKnownGateways = map[string]*gateway.PublicGateway{
 	"localhost": subdomainGatewaySpec,
 }
 
-func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification {
-	gws := map[string]*gateway.Specification{}
-
-	// First, implicit defaults such as subdomain gateway on localhost
-	for hostname, gw := range defaultKnownGateways {
-		gws[hostname] = gw
+func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
+	cfg, err := n.Repo.Config()
+	if err != nil {
+		return gateway.Config{}, err
 	}
 
-	// Then apply values from Gateway.PublicGateways, if present in the config
-	for hostname, gw := range publicGateways {
+	// Parse configuration headers and add the default Access Control Headers.
+	headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
+	for h, v := range cfg.Gateway.HTTPHeaders {
+		headers[http.CanonicalHeaderKey(h)] = v
+	}
+	gateway.AddAccessControlHeaders(headers)
+
+	// Initialize gateway configuration, with empty PublicGateways, handled after.
+	gwCfg := gateway.Config{
+		Headers:               headers,
+		DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
+		NoDNSLink:             cfg.Gateway.NoDNSLink,
+		PublicGateways:        map[string]*gateway.PublicGateway{},
+	}
+
+	// Add default implicit known gateways, such as subdomain gateway on localhost.
+	for hostname, gw := range defaultKnownGateways {
+		gwCfg.PublicGateways[hostname] = gw
+	}
+
+	// Apply values from cfg.Gateway.PublicGateways if they exist.
+	for hostname, gw := range cfg.Gateway.PublicGateways {
 		if gw == nil {
 			// Remove any implicit defaults, if present. This is useful when one
-			// wants to disable subdomain gateway on localhost etc.
-			delete(gws, hostname)
+			// wants to disable subdomain gateway on localhost, etc.
+			delete(gwCfg.PublicGateways, hostname)
 			continue
 		}
 
-		gws[hostname] = &gateway.Specification{
-			Paths:         gw.Paths,
-			NoDNSLink:     gw.NoDNSLink,
-			UseSubdomains: gw.UseSubdomains,
-			InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+		gwCfg.PublicGateways[hostname] = &gateway.PublicGateway{
+			Paths:                 gw.Paths,
+			NoDNSLink:             gw.NoDNSLink,
+			UseSubdomains:         gw.UseSubdomains,
+			InlineDNSLink:         gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+			DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses),
 		}
 	}
 
-	return gws
+	return gwCfg, nil
 }

core/corehttp/gateway_test.go

@@ -14,6 +14,7 @@ import (
 	core "github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/core/coreapi"
 	repo "github.com/ipfs/kubo/repo"
+	"github.com/stretchr/testify/assert"
 
 	iface "github.com/ipfs/boxo/coreiface"
 	nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
@@ -173,3 +174,42 @@ func TestVersion(t *testing.T) {
 		t.Fatalf("response doesn't contain protocol version:\n%s", s)
 	}
 }
+
+func TestDeserializedResponsesInheritance(t *testing.T) {
+	for _, testCase := range []struct {
+		globalSetting          config.Flag
+		gatewaySetting         config.Flag
+		expectedGatewaySetting bool
+	}{
+		{config.True, config.Default, true},
+		{config.False, config.Default, false},
+		{config.False, config.True, true},
+		{config.True, config.False, false},
+	} {
+		c := config.Config{
+			Identity: config.Identity{
+				PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node
+			},
+			Gateway: config.Gateway{
+				DeserializedResponses: testCase.globalSetting,
+				PublicGateways: map[string]*config.GatewaySpec{
+					"example.com": {
+						DeserializedResponses: testCase.gatewaySetting,
+					},
+				},
+			},
+		}
+		r := &repo.Mock{
+			C: c,
+			D: syncds.MutexWrap(datastore.NewMapDatastore()),
+		}
+		n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
+		assert.NoError(t, err)
+
+		gwCfg, err := getGatewayConfig(n)
+		assert.NoError(t, err)
+
+		assert.Contains(t, gwCfg.PublicGateways, "example.com")
+		assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses)
+	}
+}

core/node/groups.go

@@ -304,9 +304,9 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 		LibP2P(bcfg, cfg, userResourceOverrides),
 		OnlineProviders(
 			cfg.Experimental.StrategicProviding,
-			cfg.Experimental.AcceleratedDHTClient,
 			cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
 			cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
+			cfg.Routing.AcceleratedDHTClient,
 		),
 	)
 }
@@ -320,12 +320,7 @@ func Offline(cfg *config.Config) fx.Option {
 		fx.Provide(libp2p.Routing),
 		fx.Provide(libp2p.ContentRouting),
 		fx.Provide(libp2p.OfflineRouting),
-		OfflineProviders(
-			cfg.Experimental.StrategicProviding,
-			cfg.Experimental.AcceleratedDHTClient,
-			cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
-			cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
-		),
+		OfflineProviders(),
 	)
 }
 

core/node/libp2p/rcmgr.go

@@ -23,6 +23,8 @@ import (
 	"github.com/ipfs/kubo/repo"
 )
 
+var rcmgrLogger = logging.Logger("rcmgr")
+
 const NetLimitTraceFilename = "rcmgr.json.gz"
 
 var ErrNoResourceMgr = fmt.Errorf("missing ResourceMgr: make sure the daemon is running with Swarm.ResourceMgr.Enabled")
@@ -56,14 +58,13 @@ func ResourceManager(cfg config.SwarmConfig, userResourceOverrides rcmgr.Partial
 			}
 
 			if !isPartialConfigEmpty(userResourceOverrides) {
-				fmt.Print(`
+				rcmgrLogger.Info(`
 libp2p-resource-limit-overrides.json has been loaded, "default" fields will be
-filled in with autocomputed defaults.
-`)
+filled in with autocomputed defaults.`)
 			}
 
 			// We want to see this message on startup, that's why we are using fmt instead of log.
-			fmt.Print(msg)
+			rcmgrLogger.Info(msg)
 
 			if err := ensureConnMgrMakeSenseVsResourceMgr(limitConfig, cfg); err != nil {
 				return nil, opts, err
@@ -109,7 +110,7 @@ filled in with autocomputed defaults.
 			lrm.start(helpers.LifecycleCtx(mctx, lc))
 			manager = lrm
 		} else {
-			fmt.Println("go-libp2p resource manager protection disabled")
+			rcmgrLogger.Info("go-libp2p resource manager protection disabled")
 			manager = &network.NullResourceManager{}
 		}
 

core/node/libp2p/routing.go

@@ -90,7 +90,7 @@ func BaseRouting(cfg *config.Config) interface{} {
 			}
 		}
 
-		if dualDHT != nil && cfg.Experimental.AcceleratedDHTClient {
+		if dualDHT != nil && cfg.Routing.AcceleratedDHTClient {
 			cfg, err := in.Repo.Config()
 			if err != nil {
 				return out, err

core/node/libp2p/routingopt.go

@@ -47,7 +47,7 @@ func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.Parallel
 	var routers []*routinghelpers.ParallelRouter
 	// Append HTTP routers for additional speed
 	for _, endpoint := range defaultHTTPRouters {
-		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, cfg.Addresses.Swarm, cfg.Identity.PrivKey)
+		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
 		if err != nil {
 			return nil, err
 		}
@@ -123,7 +123,7 @@ func constructDHTRouting(mode dht.ModeOpt) RoutingOption {
 }
 
 // ConstructDelegatedRouting is used when Routing.Type = "custom"
-func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs []string, privKey string) RoutingOption {
+func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string) RoutingOption {
 	return func(args RoutingOptionArgs) (routing.Routing, error) {
 		return irouting.Parse(routers, methods,
 			&irouting.ExtraDHTParams{
@@ -135,7 +135,7 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
 			},
 			&irouting.ExtraHTTPParams{
 				PeerID:     peerID,
-				Addrs:      addrs,
+				Addrs:      httpAddrsFromConfig(addrs),
 				PrivKeyB64: privKey,
 			})
 	}
@@ -151,3 +151,31 @@ var (
 	DHTServerOption               = constructDHTRouting(dht.ModeServer)
 	NilRouterOption               = constructNilRouting
 )
+
+// httpAddrsFromConfig creates a list of addresses from the provided configuration to be used by HTTP delegated routers.
+func httpAddrsFromConfig(cfgAddrs config.Addresses) []string {
+	// Swarm addrs are announced by default
+	addrs := cfgAddrs.Swarm
+	// if Announce addrs are specified - override Swarm
+	if len(cfgAddrs.Announce) > 0 {
+		addrs = cfgAddrs.Announce
+	} else if len(cfgAddrs.NoAnnounce) > 0 {
+		// if Announce adds are not specified - filter Swarm addrs with NoAnnounce list
+		maddrs := map[string]struct{}{}
+		for _, addr := range addrs {
+			maddrs[addr] = struct{}{}
+		}
+		for _, addr := range cfgAddrs.NoAnnounce {
+			delete(maddrs, addr)
+		}
+		addrs = make([]string, 0, len(maddrs))
+		for k := range maddrs {
+			addrs = append(addrs, k)
+		}
+	}
+	// append AppendAnnounce addrs to the result list
+	if len(cfgAddrs.AppendAnnounce) > 0 {
+		addrs = append(addrs, cfgAddrs.AppendAnnounce...)
+	}
+	return addrs
+}

core/node/libp2p/routingopt_test.go

@@ -0,0 +1,34 @@
+package libp2p
+
+import (
+	"testing"
+
+	config "github.com/ipfs/kubo/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHttpAddrsFromConfig(t *testing.T) {
+	require.Equal(t, []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic"},
+		httpAddrsFromConfig(config.Addresses{
+			Swarm: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic"},
+		}), "Swarm addrs should be taken by default")
+
+	require.Equal(t, []string{"/ip4/192.168.0.1/tcp/4001"},
+		httpAddrsFromConfig(config.Addresses{
+			Swarm:    []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic"},
+			Announce: []string{"/ip4/192.168.0.1/tcp/4001"},
+		}), "Announce addrs should override Swarm if specified")
+
+	require.Equal(t, []string{"/ip4/0.0.0.0/udp/4001/quic"},
+		httpAddrsFromConfig(config.Addresses{
+			Swarm:      []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic"},
+			NoAnnounce: []string{"/ip4/0.0.0.0/tcp/4001"},
+		}), "Swarm addrs should not contain NoAnnounce addrs")
+
+	require.Equal(t, []string{"/ip4/192.168.0.1/tcp/4001", "/ip4/192.168.0.2/tcp/4001"},
+		httpAddrsFromConfig(config.Addresses{
+			Swarm:          []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic"},
+			Announce:       []string{"/ip4/192.168.0.1/tcp/4001"},
+			AppendAnnounce: []string{"/ip4/192.168.0.2/tcp/4001"},
+		}), "AppendAnnounce addrs should be included if specified")
+}

core/node/provider.go

@@ -5,145 +5,158 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/ipfs/boxo/blockstore"
 	"github.com/ipfs/boxo/fetcher"
 	pin "github.com/ipfs/boxo/pinning/pinner"
 	provider "github.com/ipfs/boxo/provider"
-	"github.com/ipfs/boxo/provider/batched"
-	q "github.com/ipfs/boxo/provider/queue"
-	"github.com/ipfs/boxo/provider/simple"
-	"go.uber.org/fx"
-
-	"github.com/ipfs/kubo/core/node/helpers"
 	"github.com/ipfs/kubo/repo"
 	irouting "github.com/ipfs/kubo/routing"
+	"go.uber.org/fx"
 )
 
-// SIMPLE
-
-// ProviderQueue creates new datastore backed provider queue
-func ProviderQueue(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (*q.Queue, error) {
-	return q.NewQueue(helpers.LifecycleCtx(mctx, lc), "provider-v1", repo.Datastore())
-}
-
-// SimpleProvider creates new record provider
-func SimpleProvider(mctx helpers.MetricsCtx, lc fx.Lifecycle, queue *q.Queue, rt irouting.ProvideManyRouter) provider.Provider {
-	return simple.NewProvider(helpers.LifecycleCtx(mctx, lc), queue, rt)
-}
-
-// SimpleReprovider creates new reprovider
-func SimpleReprovider(reproviderInterval time.Duration) interface{} {
-	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, rt irouting.ProvideManyRouter, keyProvider simple.KeyChanFunc) (provider.Reprovider, error) {
-		return simple.NewReprovider(helpers.LifecycleCtx(mctx, lc), reproviderInterval, rt, keyProvider), nil
-	}
-}
-
-// SimpleProviderSys creates new provider system
-func SimpleProviderSys(isOnline bool) interface{} {
-	return func(lc fx.Lifecycle, p provider.Provider, r provider.Reprovider) provider.System {
-		sys := provider.NewSystem(p, r)
-
-		if isOnline {
-			lc.Append(fx.Hook{
-				OnStart: func(ctx context.Context) error {
-					sys.Run()
-					return nil
-				},
-				OnStop: func(ctx context.Context) error {
-					return sys.Close()
-				},
-			})
+func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option {
+	const magicThroughputReportCount = 128
+	return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) {
+		opts := []provider.Option{
+			provider.Online(cr),
+			provider.ReproviderInterval(reprovideInterval),
+			provider.KeyProvider(keyProvider),
 		}
+		if !acceleratedDHTClient {
+			// The estimation kinda suck if you are running with accelerated DHT client,
+			// given this message is just trying to push people to use the acceleratedDHTClient
+			// let's not report on through if it's in use
+			opts = append(opts,
+				provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool {
+					avgProvideSpeed := duration / time.Duration(keysProvided)
+					count := uint64(keysProvided)
 
-		return sys
-	}
-}
+					if !reprovide || !complete {
+						// We don't know how many CIDs we have to provide, try to fetch it from the blockstore.
+						// But don't try for too long as this might be very expensive if you have a huge datastore.
+						ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+						defer cancel()
 
-// BatchedProviderSys creates new provider system
-func BatchedProviderSys(isOnline bool, reprovideInterval time.Duration) interface{} {
-	return func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) {
-		sys, err := batched.New(cr, q,
-			batched.ReproviderInterval(reprovideInterval),
-			batched.Datastore(repo.Datastore()),
-			batched.KeyProvider(keyProvider))
+						// FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup.
+						ch, err := bs.AllKeysChan(ctx)
+						if err != nil {
+							logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err)
+							return false
+						}
+						count = 0
+					countLoop:
+						for {
+							select {
+							case _, ok := <-ch:
+								if !ok {
+									break countLoop
+								}
+								count++
+							case <-ctx.Done():
+								// really big blockstore mode
+
+								// how many blocks would be in a 10TiB blockstore with 128KiB blocks.
+								const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024)
+								// How long per block that lasts us.
+								expectedProvideSpeed := reprovideInterval / probableBigBlockstore
+								if avgProvideSpeed > expectedProvideSpeed {
+									logger.Errorf(`
+🔔🔔🔔 YOU MAY BE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
+
+⚠️ Your system might be struggling to keep up with DHT reprovides!
+This means your content could partially or completely inaccessible on the network.
+We observed that you recently provided %d keys at an average rate of %v per key.
+
+🕑 An attempt to estimate your blockstore size timed out after 5 minutes,
+implying your blockstore might be exceedingly large. Assuming a considerable
+size of 10TiB, it would take %v to provide the complete set.
+
+⏰ The total provide time needs to stay under your reprovide interval (%v) to prevent falling behind!
+
+💡 Consider enabling the Accelerated DHT to enhance your system performance. See:
+https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`,
+										keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore, reprovideInterval)
+									return false
+								}
+							}
+						}
+					}
+
+					// How long per block that lasts us.
+					expectedProvideSpeed := reprovideInterval / time.Duration(count)
+					if avgProvideSpeed > expectedProvideSpeed {
+						// FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore.
+						logger.Errorf(`
+🔔🔔🔔 YOU ARE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
+
+⚠️ Your system is struggling to keep up with DHT reprovides!
+This means your content could partially or completely inaccessible on the network.
+We observed that you recently provided %d keys at an average rate of %v per key.
+
+💾 Your total CID count is ~%d which would total at %v reprovide process.
+
+⏰ The total provide time needs to stay under your reprovide interval (%v) to prevent falling behind!
+
+💡 Consider enabling the Accelerated DHT to enhance your reprovide throughput. See:
+https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`,
+							keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count), reprovideInterval)
+					}
+					return false
+				}, magicThroughputReportCount))
+		}
+		sys, err := provider.New(repo.Datastore(), opts...)
 		if err != nil {
 			return nil, err
 		}
 
-		if isOnline {
-			lc.Append(fx.Hook{
-				OnStart: func(ctx context.Context) error {
-					sys.Run()
-					return nil
-				},
-				OnStop: func(ctx context.Context) error {
-					return sys.Close()
-				},
-			})
-		}
+		lc.Append(fx.Hook{
+			OnStop: func(ctx context.Context) error {
+				return sys.Close()
+			},
+		})
 
 		return sys, nil
-	}
+	})
 }
 
 // ONLINE/OFFLINE
 
 // OnlineProviders groups units managing provider routing records online
-func OnlineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option {
+func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option {
 	if useStrategicProviding {
-		return fx.Provide(provider.NewOfflineProvider)
+		return OfflineProviders()
 	}
 
-	return fx.Options(
-		SimpleProviders(reprovideStrategy, reprovideInterval),
-		maybeProvide(SimpleProviderSys(true), !useBatchedProviding),
-		maybeProvide(BatchedProviderSys(true, reprovideInterval), useBatchedProviding),
-	)
-}
-
-// OfflineProviders groups units managing provider routing records offline
-func OfflineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option {
-	if useStrategicProviding {
-		return fx.Provide(provider.NewOfflineProvider)
-	}
-
-	return fx.Options(
-		SimpleProviders(reprovideStrategy, reprovideInterval),
-		maybeProvide(SimpleProviderSys(false), true),
-		//maybeProvide(BatchedProviderSys(false, reprovideInterval), useBatchedProviding),
-	)
-}
-
-// SimpleProviders creates the simple provider/reprovider dependencies
-func SimpleProviders(reprovideStrategy string, reproviderInterval time.Duration) fx.Option {
 	var keyProvider fx.Option
 	switch reprovideStrategy {
-	case "all":
-		fallthrough
-	case "":
-		keyProvider = fx.Provide(simple.NewBlockstoreProvider)
+	case "all", "":
+		keyProvider = fx.Provide(provider.NewBlockstoreProvider)
 	case "roots":
 		keyProvider = fx.Provide(pinnedProviderStrategy(true))
 	case "pinned":
 		keyProvider = fx.Provide(pinnedProviderStrategy(false))
 	default:
-		return fx.Error(fmt.Errorf("unknown reprovider strategy '%s'", reprovideStrategy))
+		return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
 	}
 
 	return fx.Options(
-		fx.Provide(ProviderQueue),
-		fx.Provide(SimpleProvider),
 		keyProvider,
-		fx.Provide(SimpleReprovider(reproviderInterval)),
+		ProviderSys(reprovideInterval, acceleratedDHTClient),
 	)
 }
 
+// OfflineProviders groups units managing provider routing records offline
+func OfflineProviders() fx.Option {
+	return fx.Provide(provider.NewNoopProvider)
+}
+
 func pinnedProviderStrategy(onlyRoots bool) interface{} {
 	type input struct {
 		fx.In
 		Pinner      pin.Pinner
 		IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
 	}
-	return func(in input) simple.KeyChanFunc {
-		return simple.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher)
+	return func(in input) provider.KeyChanFunc {
+		return provider.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher)
 	}
 }

docs/RELEASE_ISSUE_TEMPLATE.md

@@ -63,6 +63,7 @@ This section covers tasks to be done ahead of the release.
     - open an access request in the [pldw](https://github.com/protocol/pldw/issues/new/choose)
     - [example](https://github.com/protocol/pldw/issues/158)
   - [ ] [kuboreleaser](https://github.com/ipfs/kuboreleaser) checked out on your system (_only if you're using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_)
+  - [ ] [Thunderdome](https://github.com/ipfs-shipyard/thunderdome) checked out on your system and configured (see the [Thunderdome release docs](./releases_thunderdome.md) for setup)
   - [ ] [docker](https://docs.docker.com/get-docker/) installed on your system (_only if you're using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_)
   - [ ] [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) installed on your system (_only if you're **NOT** using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_)
   - [ ] [zsh](https://github.com/ohmyzsh/ohmyzsh/wiki/Installing-ZSH#install-and-set-up-zsh-as-default) installed on your system
@@ -102,6 +103,8 @@ This section covers tasks to be done during each release.
     - do **NOT** use `Squash and merge` nor `Rebase and merge` because we need to be able to sign the merge commit
     - do **NOT** delete the `release-vX.Y` branch
   </details>
+- [ ] Run Thunderdome testing, see the [Thunderdome release docs](./releases_thunderdome.md) for details
+  - [ ] create a PR and merge the experiment config into Thunderdome
 - [ ] Create the release tag <details><summary>using `kuboreleaser release --version vX.Y.Z(-rcN) tag` or ...</summary>
   - This is a dangerous operation! Go and Docker publishing are difficult to reverse! Have the release reviewer verify all the commands marked with ⚠️!
   - [ ] ⚠️ ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) tag the HEAD commit using `git tag -s vX.Y.Z(-RCN) -m 'Prerelease X.Y.Z(-RCN)'`

docs/changelogs/v0.21.md

@@ -0,0 +1,148 @@
+# Kubo changelog v0.21
+
+- [v0.21.0](#v0210)
+
+## v0.21.0
+
+- [Overview](#overview)
+- [🔦 Highlights](#-highlights)
+  - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
+  - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag)
+  - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client)
+  - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages)
+  - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml)
+  - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics)
+  - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default)
+- [📝 Changelog](#-changelog)
+- [👨‍👩‍👧‍👦 Contributors](#-contributors)
+
+### Overview
+
+### 🔦 Highlights
+
+#### Saving previously seen nodes for later bootstrapping
+
+Kubo now stores a subset of connected peers as backup bootstrap nodes ([kubo#8856](https://github.com/ipfs/kubo/pull/8856)).
+These nodes are used in addition to the explicitly defined bootstrappers in the
+[`Bootstrap`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bootstrap) configuration.
+
+This enhancement improves the resiliency of the system, as it eliminates the
+necessity of relying solely on the default bootstrappers operated by Protocol
+Labs for joining the public IPFS swarm. Previously, this level of robustness
+was only available in LAN contexts with [mDNS peer discovery](https://github.com/ipfs/kubo/blob/master/docs/config.md#discoverymdns)
+enabled.
+
+With this update, the same level of robustness is applied to peers that lack
+mDNS peers and solely rely on the public DHT.
+
+#### Gateway: `DeserializedResponses` config flag
+
+This release introduces the
+[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses)
+configuration flag.
+
+With this flag, one can explicitly configure whether the gateway responds to
+deserialized requests or not. By default, this flag is enabled.
+
+Disabling deserialized responses allows the
+gateway to operate
+as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
+limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
+response types:
+[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw),
+[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car),
+and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record).
+
+With deserialized responses disabled, the Kubo gateway can serve as a block
+backend for other software (like
+[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme),
+[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md)
+etc) without the usual risks associated with hosting deserialized data behind
+third-party CIDs.
+
+#### `client/rpc` migration of `go-ipfs-http-client`
+
+The [`go-ipfs-http-client`](https://github.com/ipfs/go-ipfs-http-client) RPC has
+been migrated into [`kubo/client/rpc`](../../client/rpc).
+
+With this change the two will be kept in sync, in some previous releases we
+updated the CoreAPI with new Kubo features but forgot to port thoses to the
+http-client, making it impossible to use them together with the same coreapi
+version.
+
+For smooth transition `v0.7.0` of `go-ipfs-http-client` provides updated stubs
+for Kubo `v0.21`.
+
+#### Gateway: DAG-CBOR/-JSON previews and improved error pages
+
+In this release, we improved the HTML templates of our HTTP gateway:
+
+1. You can now preview the contents of a DAG-CBOR and DAG-JSON document from your browser, as well as follow any IPLD Links ([CBOR Tag 42](https://github.com/ipld/cid-cbor/)) contained within them.
+2. The HTML directory listings now contain [updated, higher-definition icons](https://user-images.githubusercontent.com/5447088/241224419-5385793a-d3bb-40aa-8cb0-0382b5bc56a0.png).
+3. On gateway error, instead of a plain text error message, web browsers will now get a friendly HTML response with more details regarding the problem.
+
+HTML responses are returned when request's `Accept` header includes `text/html`.
+
+| DAG-CBOR Preview | Error Page |
+| ---- | ---- |
+| ![DAG-CBOR Preview](https://github.com/ipfs/boxo/assets/5447088/973f05d1-5731-4469-9da5-d1d776891899) | ![Error Page](https://github.com/ipfs/boxo/assets/5447088/14c453df-adbc-4634-b038-133121914550) |
+
+#### Gateway: subdomain redirects are now `text/html`
+
+HTTP 301 redirects [from path to subdomain](https://specs.ipfs.tech/http-gateways/subdomain-gateway/#migrating-from-path-to-subdomain-gateway)
+no longer include the target data in the body.
+The data is returned only once, with the final HTTP 200 returned from the
+target subdomain.
+
+The HTTP 301 body now includes human-readable `text/html` message
+for clients that do not follow redirects by default:
+
+```console
+$ curl "https://subdomain-gw.example.net/ipfs/${cid}/"
+<a href="http://${cid}.ipfs.subdomain-gw.example.net/">Moved Permanently</a>.
+```
+
+Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913).
+
+#### Gateway: support for CAR parameters of IPIP-402
+
+The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402).
+
+#### `ipfs dag stat` deduping statistics
+
+`ipfs dat stat` now accept multiple CIDs and will dump advanced statistics
+on the number of shared blocks and size of each CID.
+
+```console
+$ ipfs dag stat --progress=false QmfXuRxzyVy5H2LssLgtXrKCrNvDY8UBvMp2aoW8LS8AYA QmfZDyu2UFfUhL4VdHaw7Hofivmn5D4DdQj38Lwo86RsnB
+
+CID                                           	Blocks         	Size
+QmfXuRxzyVy5H2LssLgtXrKCrNvDY8UBvMp2aoW8LS8AYA	3              	2151
+QmfZDyu2UFfUhL4VdHaw7Hofivmn5D4DdQj38Lwo86RsnB	4              	3223
+
+Summary
+Total Size: 3326
+Unique Blocks: 5
+Shared Size: 2048
+Ratio: 1.615755
+```
+
+`ipfs --enc=json dag stat`'s keys are a non breaking change, new keys have been added but old keys with previous sementics are still here.
+
+#### Accelerated DHT Client is no longer experimental
+
+The [accelerated DHT client](docs/config.md#routingaccelerateddhtclient) is now
+the main recommended solution for users who are hosting lots of data.
+By trading some upfront DHT caching and increased memory usage,
+one gets provider throughput improvements up to 6 millions times bigger dataset.
+See [the docs](docs/config.md#routingaccelerateddhtclient) for more info.
+
+The `Experimental.AcceleratedDHTClient` flag moved to `[Routing.AcceleratedDHTClient](docs/config.md#routingaccelerateddhtclient)`.
+A config migration has been added to handle this automatically.
+
+A new tracker estimates the providing speed and warns users if they
+should be using AcceleratedDHTClient because they are falling behind.
+
+### 📝 Changelog
+
+### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -50,6 +50,7 @@ config file at runtime.
   - [`Gateway`](#gateway)
     - [`Gateway.NoFetch`](#gatewaynofetch)
     - [`Gateway.NoDNSLink`](#gatewaynodnslink)
+    - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses)
     - [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
     - [`Gateway.RootRedirect`](#gatewayrootredirect)
     - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
@@ -60,6 +61,7 @@ config file at runtime.
       - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains)
       - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink)
       - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink)
+      - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses)
       - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways)
     - [`Gateway` recipes](#gateway-recipes)
   - [`Identity`](#identity)
@@ -108,6 +110,7 @@ config file at runtime.
     - [`Reprovider.Strategy`](#reproviderstrategy)
   - [`Routing`](#routing)
     - [`Routing.Type`](#routingtype)
+    - [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient)
     - [`Routing.Routers`](#routingrouters)
       - [`Routing.Routers: Type`](#routingrouters-type)
       - [`Routing.Routers: Parameters`](#routingrouters-parameters)
@@ -236,7 +239,7 @@ documented in `ipfs config profile --help`.
     smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in
     your IPFS node, and disk usage is more critical than performance, consider using
     `flatfs`.
-  - This datastore uses up to several gigabytes of memory.  
+  - This datastore uses up to several gigabytes of memory.
   - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
   - The current implementation is based on old badger 1.x which is no longer supported by the upstream team.
 
@@ -646,6 +649,16 @@ Default: `false`
 
 Type: `bool`
 
+#### `Gateway.DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: `true`
+
+Type: `flag`
+
 ### `Gateway.HTTPHeaders`
 
 Headers to set on gateway responses.
@@ -790,6 +803,16 @@ Default: `false`
 
 Type: `flag`
 
+#### `Gateway.PublicGateways: DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: same as global `Gateway.DeserializedResponses`
+
+Type: `flag`
+
 #### Implicit defaults of `Gateway.PublicGateways`
 
 Default entries for `localhost` hostname and loopback IPs are always present.
@@ -895,7 +918,7 @@ Type: `string` (base64 encoded)
 
 ## `Internal`
 
-This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.  
+This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.
 
 **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.**
 
@@ -971,7 +994,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB)
 ### `Internal.Bitswap.ProviderSearchDelay`
 
 This parameter determines how long to wait before looking for providers outside of bitswap.
-Other routing systems like the DHT are able to provide results in less than a second, so lowering 
+Other routing systems like the DHT are able to provide results in less than a second, so lowering
 this number will allow faster peers lookups in some cases.
 
 Type: `optionalDuration` (`null` means default which is 1s)
@@ -1326,15 +1349,19 @@ Type: `array[peering]`
 ### `Reprovider.Interval`
 
 Sets the time between rounds of reproviding local content to the routing
-system. If unset, it defaults to 12 hours. If set to the value `"0"` it will
-disable content reproviding.
+system.
+
+- If unset, it uses the implicit safe default.
+- If set to the value `"0"` it will disable content reproviding.
 
 Note: disabling content reproviding will result in other nodes on the network
 not being able to discover that you have the objects that you have. If you want
 to have this disabled and keep the network aware of what you have, you must
 manually announce your content periodically.
 
-Type: `duration`
+Default: `22h` (`DefaultReproviderInterval`)
+
+Type: `optionalDuration` (unset for the default)
 
 ### `Reprovider.Strategy`
 
@@ -1346,7 +1373,7 @@ Tells reprovider what should be announced. Valid strategies are:
 
 Default: `"all"`
 
-Type: `string` (or unset for the default, which is "all")
+Type: `optionalString` (unset for the default)
 
 ## `Routing`
 
@@ -1397,6 +1424,53 @@ Default: `auto` (DHT + IPNI)
 
 Type: `optionalString` (`null`/missing means the default)
 
+
+### `Routing.AcceleratedDHTClient`
+
+This alternative DHT client with a Full-Routing-Table strategy will
+do a complete scan of the DHT every hour and record all nodes found.
+Then when a lookup is tried instead of having to go through multiple Kad hops it
+is able to find the 20 final nodes by looking up the in-memory recorded network table.
+
+This means sustained higher memory to store the routing table
+and extra CPU and network bandwidth for each network scan.
+However the latency of individual read/write operations should be ~10x faster
+and the provide throughput up to 6 million times faster on larger datasets!
+
+This is not compatible with `Routing.Type` `custom`. If you are using composable routers
+you can configure this individualy on each router.
+
+When it is enabled:
+- Client DHT operations (reads and writes) should complete much faster
+- The provider will now use a keyspace sweeping mode allowing to keep alive
+  CID sets that are multiple orders of magnitude larger.
+  - The standard Bucket-Routing-Table DHT will still run for the DHT server (if
+    the DHT server is enabled). This means the classical routing table will
+    still be used to answer other nodes.
+    This is critical to maintain to not harm the network.
+- The operations `ipfs stats dht` will default to showing information about the accelerated DHT client
+
+**Caveats:**
+1. Running the accelerated client likely will result in more resource consumption (connections, RAM, CPU, bandwidth)
+   - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer
+   - The resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly done in rounds
+   - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in
+     egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data
+     entering your node that you don't want to advertise, then consider using [Reprovider Strategies](#reproviderstrategy)
+     to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with
+     short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then
+     you may benefit from using [Strategic Providing](experimental-features.md#strategic-providing) to prevent advertising
+     of data that you ultimately will not have.
+2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being
+prepared. This means operations like searching the DHT for particular peers or content will not work initially.
+   - You can see if the DHT has been initially populated by running `ipfs stats dht`
+3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against
+them
+
+Default: `false`
+
+Type: `bool` (missing means `false`)
+
 ### `Routing.Routers`
 
 **EXPERIMENTAL: `Routing.Routers` configuration may change in future release**
@@ -1439,7 +1513,7 @@ HTTP:
 
 DHT:
   - `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto"
-  - `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT.
+  - `"AcceleratedDHTClient"`: Set to `true` if you want to use the acceleratedDHT.
   - `"PublicIPNetwork"`: Set to `true` to create a `WAN` DHT. Set to `false` to create a `LAN` DHT.
 
 Parallel:
@@ -1548,7 +1622,7 @@ another node, even if this other node is on a different network. This may
 trigger netscan alerts on some hosting providers or cause strain in some setups.
 
 The `server` configuration profile fills up this list with sensible defaults,
-preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, 
+preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`,
 which is the multiaddress representation of `192.168.0.0/16`) but you should always
 check settings against your own network and/or hosting provider.
 
@@ -1893,10 +1967,10 @@ Each field in this section is a `flag`.
 
 #### `Swarm.Transports.Network.TCP`
 
-[TCP](https://en.wikipedia.org/wiki/Transmission_Control_Protocol) is the most
-widely used transport by Kubo nodes. It doesn't directly support encryption
-and/or multiplexing, so libp2p will layer a security & multiplexing transport
-over it.
+[TCP](https://en.wikipedia.org/wiki/Transmission_Control_Protocol) is a simple
+and widely deployed transport, it should be compatible with most implementations
+and network configurations.  TCP doesn't directly support encryption and/or
+multiplexing, so libp2p will layer a security & multiplexing transport over it.
 
 Default: Enabled
 
@@ -1924,12 +1998,14 @@ Listen Addresses:
 
 #### `Swarm.Transports.Network.QUIC`
 
-[QUIC](https://en.wikipedia.org/wiki/QUIC) is a UDP-based transport with
-built-in encryption and multiplexing. The primary benefits over TCP are:
+[QUIC](https://en.wikipedia.org/wiki/QUIC) is the most widely used transport by
+Kubo nodes. It is a UDP-based transport with built-in encryption and
+multiplexing. The primary benefits over TCP are:
 
-1. It doesn't require a file descriptor per connection, easing the load on the OS.
-2. It currently takes 2 round trips to establish a connection (our TCP transport
-   currently takes 6).
+1. It takes 1 round trip to establish a connection (our TCP transport
+   currently takes 4).
+2. No [Head-of-Line blocking](https://en.wikipedia.org/wiki/Head-of-line_blocking).
+3. It doesn't require a file descriptor per connection, easing the load on the OS.
 
 Default: Enabled
 

docs/customizing.md

@@ -0,0 +1,60 @@
+# Customizing Kubo
+
+You may want to customize Kubo if you want to reuse most of Kubo's machinery. This document discusses some approaches you may consider for customizing Kubo, and their tradeoffs.
+
+Some common use cases for customizing Kubo include:
+
+- Using a custom datastore for storing blocks, pins, or other Kubo metadata
+- Adding a custom data transfer protocol into Kubo
+- Customizing Kubo internals, such as adding allowlist/blocklist functionality to Bitswap
+- Adding new commands, interfaces, functionality, etc. to Kubo while reusing the libp2p swarm
+- Building on top of Kubo's configuration and config migration functionality
+
+## Summary
+This table summarizes the tradeoffs between the approaches below:
+
+|                     | [Boxo](#boxo-build-your-own-binary) | [Kubo Plugin](#kubo-plugins) | [Bespoke Extension Point](#bespoke-extension-points) | [Go Plugin](#go-plugins) | [Fork](#fork-kubo) |
+|:-------------------:|:-----------------------------------:|:----------------------------:|:----------------------------------------------------:|:------------------------:|:------------------:|
+|     Supported?      |                  ✅                  |              ✅               |                          ✅                           |            ❌             |         ❌          |
+|    Future-proof?    |                  ✅                  |              ❌               |                          ✅                           |            ❌             |         ❌          |
+| Fully customizable? |                  ✅                  |              ✅               |                          ❌                           |            ✅             |         ✅          |
+| Fast to implement?  |                  ❌                  |              ✅               |                          ✅                           |            ✅             |         ✅          |
+| Dynamic at runtime? |                  ❌                  |              ❌               |                          ✅                           |            ✅             |         ❌          |
+|  Add new commands?  |                  ❌                  |              ✅               |                          ❌                           |            ✅             |         ✅          |
+
+## Boxo: build your own binary
+The best way to reuse Kubo functionality is to pick the functionality you need directly from [Boxo](https://github.com/ipfs/boxo) and compile your own binary.
+
+Boxo's raison d'etre is to be an IPFS component toolbox to support building custom-made implementations and applications. If your use case is not easy to implement with Boxo, you may want to consider adding whatever functionality is needed to Boxo instead of customizing Kubo, so that the community can benefit. If you are interested in this option, please reach out to Boxo maintainers, who will be happy to help you scope & plan the work. See [Boxo's FAQ](https://github.com/ipfs/boxo#help) for more info.
+
+## Kubo Plugins
+Kubo plugins are a set of interfaces that may be implemented and injected into Kubo. Generally you should recompile the Kubo binary with your plugins added. A popular example of a Kubo plugin is [go-ds-s3](https://github.com/ipfs/go-ds-s3), which can be used to store blocks in Amazon S3.
+
+Some plugins, such as the `fx` plugin, allow deep customization of Kubo internals. As a result, Kubo maintainers can't guarantee backwards compatibility with these, so you may need to adapt to breaking changes when upgrading to new Kubo versions.
+
+For more information about the different types of Kubo plugins, see [plugins.md](./plugins.md).
+
+Kubo plugins can also be injected at runtime using Go plugins (see below), but these are hard to use and not well supported by Go, so we don't recommend them.
+
+## Bespoke Extension Points
+Certain Kubo functionality may have their own extension points. For example:
+
+* Kubo supports the [Routing v1](https://github.com/ipfs/specs/blob/main/routing/ROUTING_V1_HTTP.md) API for delegating content routing to external processes
+* Kubo supports the [Pinning Service API](https://github.com/ipfs/pinning-services-api-spec) for delegating pinning to external processes
+* Kubo supports [DNSLink](https://dnslink.dev/) for delegating name->CID mappings to DNS
+
+(This list is not exhaustive.)
+
+These can generally be developed and deployed as sidecars (or full external services) without modifying the Kubo binary.
+
+## Go Plugins
+Go provides [dynamic plugins](https://pkg.go.dev/plugin) which can be loaded at runtime into a Go binary.
+
+Kubo currently works with Go plugins. But using Go plugins requires that you compile the plugin using the exact same version of the Go toolchain with the same configuration (build flags, environment variables, etc.). As a result, you likely need to build Kubo and the plugins together at the same time, and at that point you may as well just compile the functionality directly into Kubo and avoid Go plugins.
+
+As a result, we don't recommend using Go plugins, and are likely to remove them in a future release of Kubo.
+
+## Fork Kubo
+The "nuclear option" is to fork Kubo into your own repo, make your changes, and periodically sync your repo with the Kubo repo. This can be a good option if your changes are significant and you can commit to keeping your repo in sync with Kubo.
+
+Kubo maintainers can't make any backwards compatibility guarantees about Kubo internals, so by choosing this option you're accepting the risk that you may need to spend more time adapting to breaking changes.

docs/examples/kubo-as-a-library/README.md

@@ -1,6 +1,6 @@
 # Use Kubo (go-ipfs) as a library to spawn a node and add a file
 
-> This tutorial is the sibling of the [js-ipfs IPFS 101 tutorial](https://github.com/ipfs-examples/js-ipfs-examples/tree/master/examples/ipfs-101#readme).
+> Note: if you are trying to customize or extend Kubo, you should read the [Customizing Kubo](../../customizing.md) doc
 
 By the end of this tutorial, you will learn how to:
 

docs/examples/kubo-as-a-library/go.mod

@@ -7,9 +7,9 @@ go 1.18
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.8.1
+	github.com/ipfs/boxo v0.10.0
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
-	github.com/libp2p/go-libp2p v0.27.3
+	github.com/libp2p/go-libp2p v0.27.5
 	github.com/multiformats/go-multiaddr v0.9.0
 )
 
@@ -21,8 +21,7 @@ require (
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -31,7 +30,7 @@ require (
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/ristretto v0.0.2 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -41,7 +40,7 @@ require (
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
@@ -79,16 +78,16 @@ require (
 	github.com/ipfs/go-ipfs-files v0.3.0 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
-	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-ipld-format v0.5.0 // indirect
 	github.com/ipfs/go-ipld-git v0.1.1 // indirect
-	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-log/v2 v2.5.1 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
-	github.com/ipfs/go-unixfsnode v1.6.0 // indirect
+	github.com/ipfs/go-unixfsnode v1.7.1 // indirect
 	github.com/ipld/edelweiss v0.2.0 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.20.0 // indirect
@@ -96,19 +95,19 @@ require (
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/klauspost/compress v1.16.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
-	github.com/libp2p/go-libp2p-kad-dht v0.23.0 // indirect
-	github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.24.0 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.6.1 // indirect
 	github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect
 	github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
 	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-routing-helpers v0.6.2 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.7.0 // indirect
 	github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
 	github.com/libp2p/go-mplex v0.7.0 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
@@ -118,12 +117,12 @@ require (
 	github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
 	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.53 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
@@ -131,8 +130,8 @@ require (
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
-	github.com/multiformats/go-multicodec v0.8.1 // indirect
-	github.com/multiformats/go-multihash v0.2.1 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
@@ -140,6 +139,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/openzipkin/zipkin-go v0.4.1 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
@@ -157,42 +157,44 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/dig v1.16.1 // indirect
-	go.uber.org/fx v1.19.2 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/fx v1.19.3 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	gonum.org/v1/gonum v0.11.0 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
+	gonum.org/v1/gonum v0.13.0 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
-	lukechampine.com/blake3 v1.1.7 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 )

docs/examples/kubo-as-a-library/go.sum

@@ -74,10 +74,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
-github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
-github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2oZiOeReJRdU=
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
@@ -123,9 +121,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU=
 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U=
-github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
+github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
@@ -176,8 +174,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -207,8 +205,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -321,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU=
-github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI=
+github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY=
+github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -331,7 +329,6 @@ github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+Dy
 github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE=
 github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -383,18 +380,18 @@ github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3
 github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8=
 github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
-github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
 github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
+github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
+github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
 github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0=
 github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
-github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
-github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ=
-github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
+github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds=
+github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M=
 github.com/ipfs/go-ipld-git v0.1.1 h1:TWGnZjS0htmEmlMFEkA3ogrNCqWjIxwr16x1OsdhG+Y=
 github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI=
-github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc=
-github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
+github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk=
+github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM=
 github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
@@ -412,19 +409,19 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
 github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
-github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
-github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
+github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
+github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
 github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d h1:22g+x1tgWSXK34i25qjs+afr7basaneEkHaglBshd2g=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
-github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
 github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
 github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
+github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
@@ -459,8 +456,8 @@ github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5
 github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
+github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
 github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0=
@@ -489,17 +486,17 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
 github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
-github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs=
-github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE=
+github.com/libp2p/go-libp2p v0.27.5 h1:KwA7pXKXpz8hG6Cr1fMA7UkgleogcwQj0sxl5qquWRg=
+github.com/libp2p/go-libp2p v0.27.5/go.mod h1:oMfQGTb9CHnrOuSM6yMmyK2lXz3qIhnkn2+oK3B1Y2g=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
 github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
-github.com/libp2p/go-libp2p-kad-dht v0.23.0 h1:sxE6LxLopp79eLeV695n7+c77V/Vn4AMF28AdM/XFqM=
-github.com/libp2p/go-libp2p-kad-dht v0.23.0/go.mod h1:oO5N308VT2msnQI6qi5M61wzPmJYg7Tr9e16m5n7uDU=
+github.com/libp2p/go-libp2p-kad-dht v0.24.0 h1:nZnFDQEFU4N8GzclnR+IGxIgR7k4PPCDk/GK9A28onk=
+github.com/libp2p/go-libp2p-kad-dht v0.24.0/go.mod h1:lfu5T01EH+r6uDZ/8G+ObhwgzVyd0b1nb54AdT8XGhc=
 github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
-github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA=
-github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U=
+github.com/libp2p/go-libp2p-kbucket v0.6.1 h1:Y/NIvALuY5/fJlOpaJor9Azg4eor15JskGs9Lb2EhH0=
+github.com/libp2p/go-libp2p-kbucket v0.6.1/go.mod h1:dvWO707Oq/vhMVuUhyfLkw0QsOrJFETepbNfpVHSELI=
 github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
 github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo=
 github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc=
@@ -507,8 +504,8 @@ github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4s
 github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
-github.com/libp2p/go-libp2p-routing-helpers v0.6.2 h1:u6SWfX+3LoqqTAFxWVl79RkcIDE3Zsay5d+JohlEBaE=
-github.com/libp2p/go-libp2p-routing-helpers v0.6.2/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8=
+github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA=
+github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8=
 github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
 github.com/libp2p/go-libp2p-xor v0.1.0 h1:hhQwT4uGrBcuAkUGXADuPltalOdpf9aag9kaYNT2tLA=
 github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY=
@@ -544,8 +541,8 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -565,8 +562,9 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -603,8 +601,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw
 github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
 github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
-github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8=
-github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
+github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
+github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
@@ -612,8 +610,8 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
-github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108=
-github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
+github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
+github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
 github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo=
 github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
 github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
@@ -649,13 +647,13 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
+github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
-github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
@@ -754,7 +752,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
@@ -785,6 +783,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
+github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
@@ -812,38 +811,40 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
+go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
 go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q=
 go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA=
-go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho=
-go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w=
 go.opentelemetry.io/otel/exporters/zipkin v1.14.0 h1:reEVE1upBF9tcujgvSqLJS0SrI7JQPaTKP4s4rymnSs=
 go.opentelemetry.io/otel/exporters/zipkin v1.14.0/go.mod h1:RcjvOAcvhzcufQP8aHmzRw1gE9g/VEZufDdo2w+s4sk=
-go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
-go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
+go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
+go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE=
+go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
+go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
+go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
-go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
-go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY=
-go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
+go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
+go.uber.org/fx v1.19.3 h1:YqMRE4+2IepTYCMOvXqQpRa+QAVdiSTnsHU4XNWBceA=
+go.uber.org/fx v1.19.3/go.mod h1:w2HrQg26ql9fLK7hlBiZ6JsRUKV+Lj/atT1KCjT8YhM=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -879,8 +880,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -961,8 +962,8 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1042,12 +1043,11 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1061,8 +1061,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1128,8 +1128,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
-gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
+gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM=
+gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1192,8 +1192,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
-google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
+google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA=
+google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1214,8 +1214,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
-google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
+google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
+google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1267,8 +1267,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
-lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
-lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
+lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=

docs/experimental-features.md

@@ -513,7 +513,7 @@ ipfs config --json Experimental.StrategicProviding true
     - [ ] provide roots
     - [ ] provide all
     - [ ] provide strategic
-    
+
 ## GraphSync
 
 ### State
@@ -546,59 +546,6 @@ Stable, enabled by default
 
 [Noise](https://github.com/libp2p/specs/tree/master/noise) libp2p transport based on the [Noise Protocol Framework](https://noiseprotocol.org/noise.html). While TLS remains the default transport in Kubo, Noise is easier to implement and is thus the "interop" transport between IPFS and libp2p implementations.
 
-## Accelerated DHT Client
-
-### In Version
-
-0.9.0
-
-### State
-
-Experimental, default-disabled.
-
-Utilizes an alternative DHT client that searches for and maintains more information about the network
-in exchange for being more performant.
-
-When it is enabled:
-- DHT operations should complete much faster than with it disabled
-- A batching reprovider system will be enabled which takes advantage of some properties of the experimental client to
-  very efficiently put provider records into the network
-- The standard DHT client (and server if enabled) are run alongside the alternative client
-- The operations `ipfs stats dht` and `ipfs stats provide` will have different outputs
-   - `ipfs stats provide` only works when the accelerated DHT client is enabled and shows various statistics regarding
-     the provider/reprovider system
-   - `ipfs stats dht` will default to showing information about the new client
-
-**Caveats:**
-1. Running the experimental client likely will result in more resource consumption (connections, RAM, CPU, bandwidth)
-   - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer
-   - Currently, the resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly
-     done in rounds
-   - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in
-     egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data
-     entering your node that you don't want to advertise consider using [Reprovider Strategies](config.md#reproviderstrategy)
-     to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with
-     short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then
-     you may benefit from using [Strategic Providing](#strategic-providing) to prevent advertising of data that you
-     ultimately will not have.
-2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being
-prepared. This means operations like searching the DHT for particular peers or content will not work
-   - You can see if the DHT has been initially populated by running `ipfs stats dht`
-3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against
-them
-
-### How to enable
-
-```
-ipfs config --json Experimental.AcceleratedDHTClient true
-```
-
-### Road to being a real feature
-
-- [ ] Needs more people to use and report on how well it works
-- [ ] Should be usable for queries (even if slower/less efficient) shortly after startup
-- [ ] Should be usable with non-WAN DHTs
-
 ## Optimistic Provide
 
 ### In Version
@@ -640,7 +587,7 @@ than the classic client.
    size estimation available the client will transparently fall back to the classic approach.
 2. The chosen peers to store the provider records might not be the actual closest ones. Measurements showed that this
    is not a problem.
-3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The 
+3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The
    reasoning here is that one out of the remaining 5 peers are very likely to time out and delay the whole process. To
    limit the number of in-flight async requests there is the second `OptimisticProvideJobsPoolSize` setting. Currently,
    this is set to 60. This means that at most 60 parallel background requests are allowed to be in-flight. If this

docs/http-rpc-clients.md

@@ -2,13 +2,13 @@
 
 Kubo provides official HTTP RPC  (`/api/v0`) clients for selected languages:
 
-- [js-kubo-rpc-client](https://github.com/ipfs/js-kubo-rpc-client) - Official JS client for talking to Kubo RPC over HTTP
-- [go-ipfs-api](https://github.com/ipfs/go-ipfs-api) - The go interface to ipfs's HTTP RPC - Follow https://github.com/ipfs/kubo/issues/9124 for coming changes.
-- [go-ipfs-http-client](https://github.com/ipfs/go-ipfs-http-client) - IPFS CoreAPI implementation using HTTP RPC - Follow https://github.com/ipfs/kubo/issues/9124 for coming changes.
+- [`js-kubo-rpc-client`](https://github.com/ipfs/js-kubo-rpc-client) - Official JS client for talking to Kubo RPC over HTTP
+- [`go-ipfs-api`](https://github.com/ipfs/go-ipfs-api) - The go interface to ipfs's HTTP RPC - Follow https://github.com/ipfs/kubo/issues/9124 for coming changes.
+- [`httpapi`](./client/rpc) (previously `go-ipfs-http-client`) - [`coreiface.CoreAPI`](https://pkg.go.dev/github.com/ipfs/boxo/coreiface#CoreAPI) implementation using HTTP RPC
 
 ## Recommended clients
 
-| Language |     Package Name    | Github Repository                           |
-|:--------:|:-------------------:|---------------------------------------------|
-| JS       | kubo-rpc-client     | https://github.com/ipfs/js-kubo-rpc-client  |
-| Go       | go-ipfs-http-client | https://github.com/ipfs/go-ipfs-http-client |
+| Language |     Package Name    | Github Repository                          |
+|:--------:|:-------------------:|--------------------------------------------|
+| JS       | kubo-rpc-client     | https://github.com/ipfs/js-kubo-rpc-client |
+| Go       | `rpc`               | [`./client/rpc`](./client/rpc)             |

docs/releases_thunderdome.md

@@ -0,0 +1,60 @@
+# Testing Kubo releases with Thunderdome
+This document is for running Thunderdome tests by release engineers as part of releasing Kubo.
+
+We use Thunderdome to replay ipfs.io gateway traffic in a controlled environment against two different versions of Kubo, and we record metrics and compare them to look for logic or performance regressions before releasing a new Kubo version.
+
+For background information about how Thunderdome works, see: https://github.com/ipfs-shipyard/thunderdome
+
+## Prerequisites
+
+* Ensure you have access to the "IPFS Stewards" vault in 1Password, which contains the requisite AWS Console and API credentials
+* Ensure you have Docker and the Docker CLI installed
+* Checkout the Thunderdome repo locally (or `git pull` to ensure it's up-to-date)
+* Install AWS CLI v2: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
+* Configure the AWS CLI
+  * Configure the credentials as described in the [Thunderdome documentation](https://github.com/ipfs-shipyard/thunderdome/blob/main/cmd/thunderdome/README.md#credentials), using the credentials from 1Password
+* Make sure the `thunderdome` binary is up-to-date: `go build ./cmd/thunderdome`
+  
+## Add & run an experiment
+
+Create a new release configuration JSON in the `experiments/` directory, based on the most recent `kubo-release` configuration, and tweak as necessary. Generally we setup the targets to run a commit against the tag of the last release, such as:
+
+```json
+	"targets": [
+		{
+			"name": "kubo190-4283b9",
+			"description": "kubo 0.19.0-rc1",
+			"build_from_git": {
+				"repo": "https://github.com/ipfs/kubo.git",
+				"commit":"4283b9d98f8438fc8751ccc840d8fc24eeae6f13"
+			}
+		},
+		{
+			"name": "kubo181",
+			"description": "kubo 0.18.",
+			"build_from_git": {
+				"repo": "https://github.com/ipfs/kubo.git",
+				"tag":"v0.18.1"
+			}
+		}
+	]
+```
+  
+Run the experiment (where `$EXPERIMENT_CONFIG_JSON` is a path to the config JSON created above):
+
+```shell
+AWS_PROFILE=thunderdome ./thunderdome deploy --verbose --duration 120 $EXPERIMENT_CONFIG_JSON
+```
+
+This will build the Docker images, upload them to ECR, and then launch the experiment in Thunderdome. Once the experiment starts, the CLI will exit and the experiment will continue to run for the duration.
+
+## Analyze Results
+
+Add a log entry in https://www.notion.so/pl-strflt/ce2d1bd56f3541028d960d3711465659 and link to it from the release issue, so that experiment results are publicly visible.
+
+The `deploy` command will output a link to the Grafana dashboard for the experiment. We don't currently have rigorous acceptance criteria, so you should look for anomalies or changes in the metrics and make sure they are tolerable and explainable. Unexplainable anomalies should be noted in the log with a screenshot, and then root caused.
+
+
+## Open a PR to merge the experiment config into Thunderdome
+
+This is important for both posterity, and so that someone else can sanity-check the test parameters.

gc/gc.go

@@ -154,7 +154,7 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
 // Descendants recursively finds all the descendants of the given roots and
 // adds them to the given cid.Set, using the provided dag.GetLinks function
 // to walk the tree.
-func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots []cid.Cid) error {
+func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots <-chan pin.StreamedCid) error {
 	verifyGetLinks := func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) {
 		err := verifcid.ValidateCid(c)
 		if err != nil {
@@ -167,7 +167,7 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
 	verboseCidError := func(err error) error {
 		if strings.Contains(err.Error(), verifcid.ErrBelowMinimumHashLength.Error()) ||
 			strings.Contains(err.Error(), verifcid.ErrPossiblyInsecureHashFunction.Error()) {
-			err = fmt.Errorf("\"%s\"\nPlease run 'ipfs pin verify'"+ //nolint
+			err = fmt.Errorf("\"%s\"\nPlease run 'ipfs pin verify'"+ // nolint
 				" to list insecure hashes. If you want to read them,"+
 				" please downgrade your go-ipfs to 0.4.13\n", err)
 			log.Error(err)
@@ -175,19 +175,29 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
 		return err
 	}
 
-	for _, c := range roots {
-		// Walk recursively walks the dag and adds the keys to the given set
-		err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool {
-			return set.Visit(toCidV1(k))
-		}, dag.Concurrent())
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case wrapper, ok := <-roots:
+			if !ok {
+				return nil
+			}
+			if wrapper.Err != nil {
+				return wrapper.Err
+			}
 
-		if err != nil {
-			err = verboseCidError(err)
-			return err
+			// Walk recursively walks the dag and adds the keys to the given set
+			err := dag.Walk(ctx, verifyGetLinks, wrapper.C, func(k cid.Cid) bool {
+				return set.Visit(toCidV1(k))
+			}, dag.Concurrent())
+
+			if err != nil {
+				err = verboseCidError(err)
+				return err
+			}
 		}
 	}
-
-	return nil
 }
 
 // toCidV1 converts any CIDv0s to CIDv1s.
@@ -217,11 +227,8 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
 		}
 		return links, nil
 	}
-	rkeys, err := pn.RecursiveKeys(ctx)
-	if err != nil {
-		return nil, err
-	}
-	err = Descendants(ctx, getLinks, gcs, rkeys)
+	rkeys := pn.RecursiveKeys(ctx)
+	err := Descendants(ctx, getLinks, gcs, rkeys)
 	if err != nil {
 		errors = true
 		select {
@@ -243,7 +250,18 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
 		}
 		return links, nil
 	}
-	err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRoots)
+	bestEffortRootsChan := make(chan pin.StreamedCid)
+	go func() {
+		defer close(bestEffortRootsChan)
+		for _, root := range bestEffortRoots {
+			select {
+			case <-ctx.Done():
+				return
+			case bestEffortRootsChan <- pin.StreamedCid{C: root}:
+			}
+		}
+	}()
+	err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRootsChan)
 	if err != nil {
 		errors = true
 		select {
@@ -253,18 +271,15 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
 		}
 	}
 
-	dkeys, err := pn.DirectKeys(ctx)
-	if err != nil {
-		return nil, err
-	}
-	for _, k := range dkeys {
-		gcs.Add(toCidV1(k))
+	dkeys := pn.DirectKeys(ctx)
+	for k := range dkeys {
+		if k.Err != nil {
+			return nil, k.Err
+		}
+		gcs.Add(toCidV1(k.C))
 	}
 
-	ikeys, err := pn.InternalPins(ctx)
-	if err != nil {
-		return nil, err
-	}
+	ikeys := pn.InternalPins(ctx)
 	err = Descendants(ctx, getLinks, gcs, ikeys)
 	if err != nil {
 		errors = true

gc/gc_test.go

@@ -0,0 +1,96 @@
+package gc
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ipfs/boxo/blockservice"
+	"github.com/ipfs/boxo/blockstore"
+	"github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	mdutils "github.com/ipfs/boxo/ipld/merkledag/test"
+	pin "github.com/ipfs/boxo/pinning/pinner"
+	"github.com/ipfs/boxo/pinning/pinner/dspinner"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	"github.com/multiformats/go-multihash"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGC(t *testing.T) {
+	ctx := context.Background()
+
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	bs := blockstore.NewGCBlockstore(blockstore.NewBlockstore(ds), blockstore.NewGCLocker())
+	bserv := blockservice.New(bs, offline.Exchange(bs))
+	dserv := merkledag.NewDAGService(bserv)
+	pinner, err := dspinner.New(ctx, ds, dserv)
+	require.NoError(t, err)
+
+	daggen := mdutils.NewDAGGenerator()
+
+	var expectedKept []multihash.Multihash
+	var expectedDiscarded []multihash.Multihash
+
+	// add some pins
+	for i := 0; i < 5; i++ {
+		// direct
+		root, _, err := daggen.MakeDagNode(dserv.Add, 0, 1)
+		require.NoError(t, err)
+		err = pinner.PinWithMode(ctx, root, pin.Direct)
+		require.NoError(t, err)
+		expectedKept = append(expectedKept, root.Hash())
+
+		// recursive
+		root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
+		require.NoError(t, err)
+		err = pinner.PinWithMode(ctx, root, pin.Recursive)
+		require.NoError(t, err)
+		expectedKept = append(expectedKept, toMHs(allCids)...)
+	}
+
+	err = pinner.Flush(ctx)
+	require.NoError(t, err)
+
+	// add more dags to be GCed
+	for i := 0; i < 5; i++ {
+		_, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
+		require.NoError(t, err)
+		expectedDiscarded = append(expectedDiscarded, toMHs(allCids)...)
+	}
+
+	// and some other as "best effort roots"
+	var bestEffortRoots []cid.Cid
+	for i := 0; i < 5; i++ {
+		root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
+		require.NoError(t, err)
+		bestEffortRoots = append(bestEffortRoots, root)
+		expectedKept = append(expectedKept, toMHs(allCids)...)
+	}
+
+	ch := GC(ctx, bs, ds, pinner, bestEffortRoots)
+	var discarded []multihash.Multihash
+	for res := range ch {
+		require.NoError(t, res.Error)
+		discarded = append(discarded, res.KeyRemoved.Hash())
+	}
+
+	allKeys, err := bs.AllKeysChan(ctx)
+	require.NoError(t, err)
+	var kept []multihash.Multihash
+	for key := range allKeys {
+		kept = append(kept, key.Hash())
+	}
+
+	require.ElementsMatch(t, expectedDiscarded, discarded)
+	require.ElementsMatch(t, expectedKept, kept)
+}
+
+func toMHs(cids []cid.Cid) []multihash.Multihash {
+	res := make([]multihash.Multihash, len(cids))
+	for i, c := range cids {
+		res[i] = c.Hash()
+	}
+	return res
+}

go.mod

@@ -5,7 +5,7 @@ require (
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2
 	github.com/benbjohnson/clock v1.3.0
 	github.com/blang/semver/v4 v4.0.0
-	github.com/cenkalti/backoff/v4 v4.2.0
+	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/ceramicnetwork/go-dag-jose v0.1.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-systemd/v22 v22.5.0
@@ -16,7 +16,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.8.1
+	github.com/ipfs/boxo v0.10.0
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0
@@ -30,14 +30,14 @@ require (
 	github.com/ipfs/go-fs-lock v0.0.7
 	github.com/ipfs/go-graphsync v0.14.4
 	github.com/ipfs/go-ipfs-cmds v0.9.0
-	github.com/ipfs/go-ipld-format v0.4.0
+	github.com/ipfs/go-ipld-format v0.5.0
 	github.com/ipfs/go-ipld-git v0.1.1
-	github.com/ipfs/go-ipld-legacy v0.1.1
+	github.com/ipfs/go-ipld-legacy v0.2.1
 	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-unixfsnode v1.6.0
+	github.com/ipfs/go-unixfsnode v1.7.1
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
@@ -45,45 +45,47 @@ require (
 	github.com/jbenet/goprocess v0.1.4
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/libp2p/go-doh-resolver v0.4.0
-	github.com/libp2p/go-libp2p v0.27.3
+	github.com/libp2p/go-libp2p v0.27.5
 	github.com/libp2p/go-libp2p-http v0.5.0
-	github.com/libp2p/go-libp2p-kad-dht v0.23.0
-	github.com/libp2p/go-libp2p-kbucket v0.5.0
+	github.com/libp2p/go-libp2p-kad-dht v0.24.0
+	github.com/libp2p/go-libp2p-kbucket v0.6.1
 	github.com/libp2p/go-libp2p-pubsub v0.9.3
 	github.com/libp2p/go-libp2p-pubsub-router v0.6.0
 	github.com/libp2p/go-libp2p-record v0.2.0
-	github.com/libp2p/go-libp2p-routing-helpers v0.6.2
+	github.com/libp2p/go-libp2p-routing-helpers v0.7.0
 	github.com/libp2p/go-libp2p-testing v0.12.0
 	github.com/libp2p/go-socket-activation v0.1.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-multiaddr v0.9.0
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multibase v0.2.0
-	github.com/multiformats/go-multicodec v0.8.1
-	github.com/multiformats/go-multihash v0.2.1
+	github.com/multiformats/go-multicodec v0.9.0
+	github.com/multiformats/go-multihash v0.2.3
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tidwall/gjson v1.14.4
 	github.com/tidwall/sjson v1.2.5
 	github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	go.opencensus.io v0.24.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0
-	go.opentelemetry.io/contrib/propagators/autoprop v0.40.0
-	go.opentelemetry.io/otel v1.14.0
-	go.opentelemetry.io/otel/sdk v1.14.0
-	go.opentelemetry.io/otel/trace v1.14.0
-	go.uber.org/dig v1.16.1
-	go.uber.org/fx v1.19.2
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
+	go.opentelemetry.io/contrib/propagators/autoprop v0.42.0
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/sdk v1.16.0
+	go.opentelemetry.io/otel/trace v1.16.0
+	go.uber.org/dig v1.17.0
+	go.uber.org/fx v1.19.3
+	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.7.0
+	golang.org/x/crypto v0.9.0
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/mod v0.10.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.7.0
+	golang.org/x/sys v0.8.0
 )
 
 require (
@@ -92,7 +94,6 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
@@ -100,7 +101,7 @@ require (
 	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/ristretto v0.0.2 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -111,7 +112,7 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
@@ -135,14 +136,14 @@ require (
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
 	github.com/ipfs/go-libipfs v0.7.0 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipld/edelweiss v0.2.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/klauspost/compress v1.16.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
@@ -159,14 +160,14 @@ require (
 	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-colorable v0.1.4 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/miekg/dns v1.1.53 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
@@ -176,6 +177,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/openzipkin/zipkin-go v0.4.1 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
@@ -196,41 +198,40 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
-	go.opentelemetry.io/contrib/propagators/aws v1.15.0 // indirect
-	go.opentelemetry.io/contrib/propagators/b3 v1.15.0 // indirect
-	go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 // indirect
-	go.opentelemetry.io/contrib/propagators/ot v1.15.0 // indirect
+	go.opentelemetry.io/contrib/propagators/aws v1.17.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.17.0 // indirect
+	go.opentelemetry.io/contrib/propagators/jaeger v1.17.0 // indirect
+	go.opentelemetry.io/contrib/propagators/ot v1.17.0 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect
-	go.opentelemetry.io/otel/metric v0.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	gonum.org/v1/gonum v0.11.0 // indirect
+	gonum.org/v1/gonum v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/blake3 v1.1.7 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 )
 

go.sum

@@ -84,10 +84,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
-github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
-github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2oZiOeReJRdU=
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
@@ -137,9 +135,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU=
 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U=
-github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
+github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
@@ -208,8 +206,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -240,8 +238,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -356,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU=
-github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI=
+github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY=
+github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -366,7 +364,6 @@ github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+Dy
 github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE=
 github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -419,18 +416,18 @@ github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3
 github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8=
 github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
-github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
 github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
+github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
+github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
 github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0=
 github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
-github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
-github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ=
-github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
+github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds=
+github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M=
 github.com/ipfs/go-ipld-git v0.1.1 h1:TWGnZjS0htmEmlMFEkA3ogrNCqWjIxwr16x1OsdhG+Y=
 github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI=
-github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc=
-github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
+github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk=
+github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM=
 github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM=
 github.com/ipfs/go-libipfs v0.7.0/go.mod h1:KsIf/03CqhICzyRGyGo68tooiBE2iFbI/rXW7FhAYr0=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
@@ -451,8 +448,8 @@ github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnz
 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
 github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
-github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
-github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
+github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
+github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
@@ -460,7 +457,6 @@ github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8=
 github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d h1:22g+x1tgWSXK34i25qjs+afr7basaneEkHaglBshd2g=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
-github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
 github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
@@ -508,8 +504,8 @@ github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5
 github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
+github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
@@ -540,8 +536,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
 github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
-github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs=
-github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE=
+github.com/libp2p/go-libp2p v0.27.5 h1:KwA7pXKXpz8hG6Cr1fMA7UkgleogcwQj0sxl5qquWRg=
+github.com/libp2p/go-libp2p v0.27.5/go.mod h1:oMfQGTb9CHnrOuSM6yMmyK2lXz3qIhnkn2+oK3B1Y2g=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
@@ -550,11 +546,11 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk
 github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA=
 github.com/libp2p/go-libp2p-http v0.5.0 h1:+x0AbLaUuLBArHubbbNRTsgWz0RjNTy6DJLOxQ3/QBc=
 github.com/libp2p/go-libp2p-http v0.5.0/go.mod h1:glh87nZ35XCQyFsdzZps6+F4HYI6DctVFY5u1fehwSg=
-github.com/libp2p/go-libp2p-kad-dht v0.23.0 h1:sxE6LxLopp79eLeV695n7+c77V/Vn4AMF28AdM/XFqM=
-github.com/libp2p/go-libp2p-kad-dht v0.23.0/go.mod h1:oO5N308VT2msnQI6qi5M61wzPmJYg7Tr9e16m5n7uDU=
+github.com/libp2p/go-libp2p-kad-dht v0.24.0 h1:nZnFDQEFU4N8GzclnR+IGxIgR7k4PPCDk/GK9A28onk=
+github.com/libp2p/go-libp2p-kad-dht v0.24.0/go.mod h1:lfu5T01EH+r6uDZ/8G+ObhwgzVyd0b1nb54AdT8XGhc=
 github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
-github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA=
-github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U=
+github.com/libp2p/go-libp2p-kbucket v0.6.1 h1:Y/NIvALuY5/fJlOpaJor9Azg4eor15JskGs9Lb2EhH0=
+github.com/libp2p/go-libp2p-kbucket v0.6.1/go.mod h1:dvWO707Oq/vhMVuUhyfLkw0QsOrJFETepbNfpVHSELI=
 github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
 github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo=
 github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc=
@@ -562,8 +558,8 @@ github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4s
 github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
-github.com/libp2p/go-libp2p-routing-helpers v0.6.2 h1:u6SWfX+3LoqqTAFxWVl79RkcIDE3Zsay5d+JohlEBaE=
-github.com/libp2p/go-libp2p-routing-helpers v0.6.2/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8=
+github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA=
+github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8=
 github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
 github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg=
 github.com/libp2p/go-libp2p-xor v0.1.0 h1:hhQwT4uGrBcuAkUGXADuPltalOdpf9aag9kaYNT2tLA=
@@ -606,8 +602,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -630,8 +626,9 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -670,8 +667,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw
 github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
 github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
-github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8=
-github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
+github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
+github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
@@ -679,8 +676,8 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
-github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108=
-github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
+github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
+github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
 github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo=
 github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
 github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
@@ -718,6 +715,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
+github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -725,7 +723,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
-github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
@@ -854,8 +851,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
@@ -898,6 +895,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
+github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
@@ -928,52 +926,52 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI=
-go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 h1:Lj33jj7eIrBfIShiK8NU91u2BglKnUS1UUxVemuQJtw=
-go.opentelemetry.io/contrib/propagators/autoprop v0.40.0/go.mod h1:6QO816FeZ+6zahs6hYqbUCCsnNBm7o+t4iwVySpzcdI=
-go.opentelemetry.io/contrib/propagators/aws v1.15.0 h1:FLe+bRTMAhEALItDQt1U2S/rdq8/rGGJTJpOpCDvMu0=
-go.opentelemetry.io/contrib/propagators/aws v1.15.0/go.mod h1:Z/nqdjqKjErrS3gYoEMZt8//dt8VZbqalD0V+7vh7lM=
-go.opentelemetry.io/contrib/propagators/b3 v1.15.0 h1:bMaonPyFcAvZ4EVzkUNkfnUHP5Zi63CIDlA3dRsEg8Q=
-go.opentelemetry.io/contrib/propagators/b3 v1.15.0/go.mod h1:VjU0g2v6HSQ+NwfifambSLAeBgevjIcqmceaKWEzl0c=
-go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 h1:xdJjwy5t/8I+TZehMMQ+r2h50HREihH2oMUhimQ+jug=
-go.opentelemetry.io/contrib/propagators/jaeger v1.15.0/go.mod h1:tU0nwW4QTvKceNUP60/PQm0FI8zDSwey7gIFt3RR/yw=
-go.opentelemetry.io/contrib/propagators/ot v1.15.0 h1:iBNejawWy7wWZ5msuZDNcMjBy14Wc0v3gCAXukGHN/Q=
-go.opentelemetry.io/contrib/propagators/ot v1.15.0/go.mod h1:0P7QQ+MHt6SXR1ATaMpewSiWlp8NbKErNLKcaU4EEKI=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8=
+go.opentelemetry.io/contrib/propagators/autoprop v0.42.0 h1:s2RzYOAqHVgG23q8fPWYChobUoZM6rJZ98EnylJr66w=
+go.opentelemetry.io/contrib/propagators/autoprop v0.42.0/go.mod h1:Mv/tWNtZn+NbALDb2XcItP0OM3lWWZjAfSroINxfW+Y=
+go.opentelemetry.io/contrib/propagators/aws v1.17.0 h1:IX8d7l2uRw61BlmZBOTQFaK+y22j6vytMVTs9wFrO+c=
+go.opentelemetry.io/contrib/propagators/aws v1.17.0/go.mod h1:pAlCYRWff4uGqRXOVn3WP8pDZ5E0K56bEoG7a1VSL4k=
+go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo=
+go.opentelemetry.io/contrib/propagators/b3 v1.17.0/go.mod h1:IkfUfMpKWmynvvE0264trz0sf32NRTZL4nuAN9AbWRc=
+go.opentelemetry.io/contrib/propagators/jaeger v1.17.0 h1:Zbpbmwav32Ea5jSotpmkWEl3a6Xvd4tw/3xxGO1i05Y=
+go.opentelemetry.io/contrib/propagators/jaeger v1.17.0/go.mod h1:tcTUAlmO8nuInPDSBVfG+CP6Mzjy5+gNV4mPxMbL0IA=
+go.opentelemetry.io/contrib/propagators/ot v1.17.0 h1:ufo2Vsz8l76eI47jFjuVyjyB3Ae2DmfiCV/o6Vc8ii0=
+go.opentelemetry.io/contrib/propagators/ot v1.17.0/go.mod h1:SbKPj5XGp8K/sGm05XblaIABgMgw2jDczP8gGeuaVLk=
+go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
+go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
 go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q=
 go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA=
-go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho=
-go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w=
 go.opentelemetry.io/otel/exporters/zipkin v1.14.0 h1:reEVE1upBF9tcujgvSqLJS0SrI7JQPaTKP4s4rymnSs=
 go.opentelemetry.io/otel/exporters/zipkin v1.14.0/go.mod h1:RcjvOAcvhzcufQP8aHmzRw1gE9g/VEZufDdo2w+s4sk=
-go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
-go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
-go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
-go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
+go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
+go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE=
+go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
+go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
+go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
-go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
-go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY=
-go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
+go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
+go.uber.org/fx v1.19.3 h1:YqMRE4+2IepTYCMOvXqQpRa+QAVdiSTnsHU4XNWBceA=
+go.uber.org/fx v1.19.3/go.mod h1:w2HrQg26ql9fLK7hlBiZ6JsRUKV+Lj/atT1KCjT8YhM=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -1009,8 +1007,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1097,8 +1095,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1109,8 +1107,8 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
-golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
+golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
+golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1193,20 +1191,19 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1216,8 +1213,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1283,8 +1280,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
-gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
+gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM=
+gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1349,8 +1346,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
-google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
+google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA=
+google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1371,8 +1368,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
-google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
+google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
+google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1428,8 +1425,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
-lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
-lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
+lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=

plugin/loader/loader.go

@@ -87,7 +87,7 @@ func (ls loaderState) String() string {
 //  5. Call Close to close all plugins.
 type PluginLoader struct {
 	state   loaderState
-	plugins map[string]plugin.Plugin
+	plugins []plugin.Plugin
 	started []plugin.Plugin
 	config  config.Plugins
 	repo    string
@@ -95,7 +95,7 @@ type PluginLoader struct {
 
 // NewPluginLoader creates new plugin loader
 func NewPluginLoader(repo string) (*PluginLoader, error) {
-	loader := &PluginLoader{plugins: make(map[string]plugin.Plugin, len(preloadPlugins)), repo: repo}
+	loader := &PluginLoader{plugins: make([]plugin.Plugin, 0, len(preloadPlugins)), repo: repo}
 	if repo != "" {
 		cfg, err := cserialize.Load(filepath.Join(repo, config.DefaultConfigFile))
 		switch err {
@@ -106,6 +106,7 @@ func NewPluginLoader(repo string) (*PluginLoader, error) {
 			return nil, err
 		}
 	}
+
 	for _, v := range preloadPlugins {
 		if err := loader.Load(v); err != nil {
 			return nil, err
@@ -140,18 +141,22 @@ func (loader *PluginLoader) Load(pl plugin.Plugin) error {
 	}
 
 	name := pl.Name()
-	if ppl, ok := loader.plugins[name]; ok {
-		// plugin is already loaded
-		return fmt.Errorf(
-			"plugin: %s, is duplicated in version: %s, "+
-				"while trying to load dynamically: %s",
-			name, ppl.Version(), pl.Version())
+
+	for _, p := range loader.plugins {
+		if p.Name() == name {
+			// plugin is already loaded
+			return fmt.Errorf(
+				"plugin: %s, is duplicated in version: %s, "+
+					"while trying to load dynamically: %s",
+				name, p.Version(), pl.Version())
+		}
 	}
+
 	if loader.config.Plugins[name].Disabled {
 		log.Infof("not loading disabled plugin %s", name)
 		return nil
 	}
-	loader.plugins[name] = pl
+	loader.plugins = append(loader.plugins, pl)
 	return nil
 }
 
@@ -219,10 +224,10 @@ func (loader *PluginLoader) Initialize() error {
 	if err := loader.transition(loaderLoading, loaderInitializing); err != nil {
 		return err
 	}
-	for name, p := range loader.plugins {
+	for _, p := range loader.plugins {
 		err := p.Init(&plugin.Environment{
 			Repo:   loader.repo,
-			Config: loader.config.Plugins[name].Config,
+			Config: loader.config.Plugins[p.Name()].Config,
 		})
 		if err != nil {
 			loader.state = loaderFailed

repo/fsrepo/fsrepo.go

@@ -37,7 +37,7 @@ const LockFile = "repo.lock"
 var log = logging.Logger("fsrepo")
 
 // RepoVersion is the version number that we are currently expecting to see
-var RepoVersion = 13
+var RepoVersion = 14
 
 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
 Sorry for the inconvenience. In the future, these will run automatically.`

repo/fsrepo/migrations/fetcher.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	// Current distribution to fetch migrations from
-	CurrentIpfsDist = "/ipfs/Qmf4yftD4LuMo8JMNPqqw3BtUwYd2VkXMiAThuPE6usrbQ" // fs-repo-12-to-13 v1.0.0
+	CurrentIpfsDist = "/ipfs/QmYerugGRCZWA8yQMKDsd9daEVXUR3C5nuw3VXuX1mggHa" // fs-repo-13-to-14 v1.0.0
 	// Latest distribution path.  Default for fetchers.
 	LatestIpfsDist = "/ipns/dist.ipfs.tech"
 

routing/composer.go

@@ -51,7 +51,7 @@ func (c *Composer) ProvideMany(ctx context.Context, keys []multihash.Multihash)
 
 func (c *Composer) Ready() bool {
 	log.Debug("composer: calling ready")
-	pmr, ok := c.ProvideRouter.(routinghelpers.ProvideManyRouter)
+	pmr, ok := c.ProvideRouter.(routinghelpers.ReadyAbleRouter)
 	if !ok {
 		return true
 	}

test/cli/backup_bootstrap_test.go

@@ -0,0 +1,60 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ipfs/kubo/config"
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBackupBootstrapPeers(t *testing.T) {
+	nodes := harness.NewT(t).NewNodes(3).Init()
+	nodes.ForEachPar(func(n *harness.Node) {
+		n.UpdateConfig(func(cfg *config.Config) {
+			cfg.Bootstrap = []string{}
+			cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", harness.NewRandPort())}
+			cfg.Discovery.MDNS.Enabled = false
+			cfg.Internal.BackupBootstrapInterval = config.NewOptionalDuration(250 * time.Millisecond)
+		})
+	})
+
+	// Start all nodes and ensure they all have no peers.
+	nodes.StartDaemons()
+	nodes.ForEachPar(func(n *harness.Node) {
+		assert.Len(t, n.Peers(), 0)
+	})
+
+	// Connect nodes 0 and 1, ensure they know each other.
+	nodes[0].Connect(nodes[1])
+	assert.Len(t, nodes[0].Peers(), 1)
+	assert.Len(t, nodes[1].Peers(), 1)
+	assert.Len(t, nodes[2].Peers(), 0)
+
+	// Wait a bit to ensure that 0 and 1 saved their temporary bootstrap backups.
+	time.Sleep(time.Millisecond * 500)
+	nodes.StopDaemons()
+
+	// Start 1 and 2. 2 does not know anyone yet.
+	nodes[1].StartDaemon()
+	nodes[2].StartDaemon()
+	assert.Len(t, nodes[1].Peers(), 0)
+	assert.Len(t, nodes[2].Peers(), 0)
+
+	// Connect 1 and 2, ensure they know each other.
+	nodes[1].Connect(nodes[2])
+	assert.Len(t, nodes[1].Peers(), 1)
+	assert.Len(t, nodes[2].Peers(), 1)
+
+	// Start 0, wait a bit. Should connect to 1, and then discover 2 via the
+	// backup bootstrap peers.
+	nodes[0].StartDaemon()
+	time.Sleep(time.Millisecond * 500)
+
+	// Check if they're all connected.
+	assert.Len(t, nodes[0].Peers(), 2)
+	assert.Len(t, nodes[1].Peers(), 2)
+	assert.Len(t, nodes[2].Peers(), 2)
+}

test/cli/content_routing_http_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/ipfs/boxo/routing/http/server"
 	"github.com/ipfs/boxo/routing/http/types"
+	"github.com/ipfs/boxo/routing/http/types/iter"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/kubo/test/cli/harness"
 	"github.com/ipfs/kubo/test/cli/testutils"
@@ -23,11 +24,11 @@ type fakeHTTPContentRouter struct {
 	provideCalls       int
 }
 
-func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error) {
+func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) {
 	r.m.Lock()
 	defer r.m.Unlock()
 	r.findProvidersCalls++
-	return []types.ProviderResponse{}, nil
+	return iter.FromSlice([]iter.Result[types.ProviderResponse]{}), nil
 }
 
 func (r *fakeHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {

test/cli/dag_test.go

@@ -0,0 +1,105 @@
+package cli
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/ipfs/kubo/test/cli/testutils"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	fixtureFile    = "./fixtures/TestDagStat.car"
+	textOutputPath = "./fixtures/TestDagStatExpectedOutput.txt"
+	node1Cid       = "bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy"
+	node2Cid       = "bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4"
+	fixtureCid     = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq"
+)
+
+type DagStat struct {
+	Cid       string `json:"Cid"`
+	Size      int    `json:"Size"`
+	NumBlocks int    `json:"NumBlocks"`
+}
+
+type Data struct {
+	UniqueBlocks int       `json:"UniqueBlocks"`
+	TotalSize    int       `json:"TotalSize"`
+	SharedSize   int       `json:"SharedSize"`
+	Ratio        float64   `json:"Ratio"`
+	DagStats     []DagStat `json:"DagStats"`
+}
+
+// The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 7B
+// when traversing the DAG from the root's children (node1 and node2) we count (46 + 7)x2 bytes (counting redundant bytes) = 106
+// since both nodes share a common child of 7 bytes we actually had to read (46)x2 + 7 =  99 bytes
+// we should get a dedup ratio of 106/99 that results in approximatelly 1.0707071
+
+func TestDag(t *testing.T) {
+	t.Parallel()
+
+	t.Run("ipfs dag stat --enc=json", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init().StartDaemon()
+		// Import fixture
+		r, err := os.Open(fixtureFile)
+		assert.Nil(t, err)
+		defer r.Close()
+		err = node.IPFSDagImport(r, fixtureCid)
+		assert.NoError(t, err)
+		stat := node.RunIPFS("dag", "stat", "--progress=false", "--enc=json", node1Cid, node2Cid)
+		var data Data
+		err = json.Unmarshal(stat.Stdout.Bytes(), &data)
+		assert.NoError(t, err)
+
+		expectedUniqueBlocks := 3
+		expectedSharedSize := 7
+		expectedTotalSize := 99
+		expectedRatio := float64(expectedSharedSize+expectedTotalSize) / float64(expectedTotalSize)
+		expectedDagStatsLength := 2
+		// Validate UniqueBlocks
+		assert.Equal(t, expectedUniqueBlocks, data.UniqueBlocks)
+		assert.Equal(t, expectedSharedSize, data.SharedSize)
+		assert.Equal(t, expectedTotalSize, data.TotalSize)
+		assert.Equal(t, testutils.FloatTruncate(expectedRatio, 4), testutils.FloatTruncate(data.Ratio, 4))
+
+		// Validate DagStats
+		assert.Equal(t, expectedDagStatsLength, len(data.DagStats))
+		node1Output := data.DagStats[0]
+		node2Output := data.DagStats[1]
+
+		assert.Equal(t, node1Output.Cid, node1Cid)
+		assert.Equal(t, node2Output.Cid, node2Cid)
+
+		expectedNode1Size := (expectedTotalSize + expectedSharedSize) / 2
+		expectedNode2Size := (expectedTotalSize + expectedSharedSize) / 2
+		assert.Equal(t, expectedNode1Size, node1Output.Size)
+		assert.Equal(t, expectedNode2Size, node2Output.Size)
+
+		expectedNode1Blocks := 2
+		expectedNode2Blocks := 2
+		assert.Equal(t, expectedNode1Blocks, node1Output.NumBlocks)
+		assert.Equal(t, expectedNode2Blocks, node2Output.NumBlocks)
+	})
+
+	t.Run("ipfs dag stat", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init().StartDaemon()
+		r, err := os.Open(fixtureFile)
+		assert.NoError(t, err)
+		defer r.Close()
+		f, err := os.Open(textOutputPath)
+		assert.NoError(t, err)
+		defer f.Close()
+		content, err := io.ReadAll(f)
+		assert.NoError(t, err)
+		err = node.IPFSDagImport(r, fixtureCid)
+		assert.NoError(t, err)
+		stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid)
+		assert.Equal(t, content, stat.Stdout.Bytes())
+	})
+
+}

test/cli/delegated_routing_http_test.go

@@ -15,17 +15,20 @@ func TestHTTPDelegatedRouting(t *testing.T) {
 	t.Parallel()
 	node := harness.NewT(t).NewNode().Init().StartDaemon()
 
-	fakeServer := func(resp string) *httptest.Server {
+	fakeServer := func(contentType string, resp ...string) *httptest.Server {
 		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			_, err := w.Write([]byte(resp))
-			if err != nil {
-				panic(err)
+			w.Header().Set("Content-Type", contentType)
+			for _, r := range resp {
+				_, err := w.Write([]byte(r))
+				if err != nil {
+					panic(err)
+				}
 			}
 		}))
 	}
 
 	findProvsCID := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq"
-	prov := "12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ"
+	provs := []string{"12D3KooWAobjw92XDcnQ1rRmRJDA3zAQpdPYUpZKrJxH6yccSpje", "12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ"}
 
 	t.Run("default routing config has no routers defined", func(t *testing.T) {
 		assert.Nil(t, node.ReadConfig().Routing.Routers)
@@ -84,11 +87,11 @@ func TestHTTPDelegatedRouting(t *testing.T) {
 	})
 
 	t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works", func(t *testing.T) {
-		server := fakeServer(ToJSONStr(JSONObj{
+		server := fakeServer("application/json", ToJSONStr(JSONObj{
 			"Providers": []JSONObj{{
 				"Protocol": "transport-bitswap",
 				"Schema":   "bitswap",
-				"ID":       prov,
+				"ID":       provs[0],
 				"Addrs":    []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"},
 			}},
 		}))
@@ -113,9 +116,39 @@ func TestHTTPDelegatedRouting(t *testing.T) {
 		assert.Equal(t, res.Stdout.Trimmed(), server.URL)
 
 		node.StartDaemon()
-
 		res = node.IPFS("routing", "findprovs", findProvsCID)
-		assert.Equal(t, prov, res.Stdout.Trimmed())
+		assert.Equal(t, provs[0], res.Stdout.Trimmed())
+	})
+
+	node.StopDaemon()
+
+	t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works (streaming)", func(t *testing.T) {
+		server := fakeServer("application/x-ndjson", ToJSONStr(JSONObj{
+			"Protocol": "transport-bitswap",
+			"Schema":   "bitswap",
+			"ID":       provs[1],
+			"Addrs":    []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"},
+		}), ToJSONStr(JSONObj{
+			"Protocol": "transport-bitswap",
+			"Schema":   "bitswap",
+			"ID":       provs[0],
+			"Addrs":    []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"},
+		}))
+		t.Cleanup(server.Close)
+
+		node.IPFS("config", "Routing.Routers.TestDelegatedRouter", "--json", ToJSONStr(JSONObj{
+			"Type": "http",
+			"Parameters": JSONObj{
+				"Endpoint": server.URL,
+			},
+		}))
+
+		res := node.IPFS("config", "Routing.Routers.TestDelegatedRouter.Parameters.Endpoint")
+		assert.Equal(t, res.Stdout.Trimmed(), server.URL)
+
+		node.StartDaemon()
+		res = node.IPFS("routing", "findprovs", findProvsCID)
+		assert.Equal(t, provs[1]+"\n"+provs[0], res.Stdout.Trimmed())
 	})
 
 	t.Run("HTTP client should emit OpenCensus metrics", func(t *testing.T) {

test/cli/fixtures/TestDagStatExpectedOutput.txt

@@ -0,0 +1,12 @@
+
+CID                                                        	Blocks         	Size
+bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy	2              	53
+bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4	2              	53
+
+Summary
+Total Size: 99
+Unique Blocks: 3
+Shared Size: 7
+Ratio: 1.070707
+
+

test/cli/gateway_range_test.go

@@ -31,10 +31,10 @@ func TestGatewayHAMTDirectory(t *testing.T) {
 
 	// Import fixtures
 	r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	defer r.Close()
 	err = node.IPFSDagImport(r, fixtureCid)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	// Fetch HAMT directory succeeds with minimal refs
 	resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid))
@@ -60,10 +60,10 @@ func TestGatewayMultiRange(t *testing.T) {
 
 	// Import fixtures
 	r, err := os.Open("./fixtures/TestGatewayMultiRange.car")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	defer r.Close()
 	err = node.IPFSDagImport(r, fixtureCid)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	// Succeeds fetching a range of blocks we have
 	resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) {

test/cli/gateway_test.go

@@ -13,8 +13,10 @@ import (
 	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/test/cli/harness"
 	. "github.com/ipfs/kubo/test/cli/testutils"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
+	"github.com/multiformats/go-multibase"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,10 +27,13 @@ func TestGateway(t *testing.T) {
 	node := h.NewNode().Init().StartDaemon("--offline")
 	cid := node.IPFSAddStr("Hello Worlds!")
 
+	peerID, err := peer.ToCid(node.PeerID()).StringOfBase(multibase.Base36)
+	assert.NoError(t, err)
+
 	client := node.GatewayClient()
 	client.TemplateData = map[string]string{
 		"CID":    cid,
-		"PeerID": node.PeerID().String(),
+		"PeerID": peerID,
 	}
 
 	t.Run("GET IPFS path succeeds", func(t *testing.T) {
@@ -182,7 +187,7 @@ func TestGateway(t *testing.T) {
 		t.Run("GET /ipfs/ipns/{peerid} returns redirect to the valid path", func(t *testing.T) {
 			t.Parallel()
 			resp := client.Get("/ipfs/ipns/{{.PeerID}}?query=to-remember")
-			peerID := node.PeerID().String()
+
 			assert.Contains(t,
 				resp.Body,
 				fmt.Sprintf(`<meta http-equiv="refresh" content="10;url=/ipns/%s?query=to-remember" />`, peerID),
@@ -474,6 +479,9 @@ func TestGateway(t *testing.T) {
 			cfg.Gateway.NoFetch = true
 		})
 
+		node2PeerID, err := peer.ToCid(node2.PeerID()).StringOfBase(multibase.Base36)
+		assert.NoError(t, err)
+
 		nodes.StartDaemons().Connect()
 
 		t.Run("not present", func(t *testing.T) {
@@ -486,7 +494,7 @@ func TestGateway(t *testing.T) {
 
 			t.Run("not present IPNS key from node 1", func(t *testing.T) {
 				t.Parallel()
-				assert.Equal(t, 500, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode)
+				assert.Equal(t, 500, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode)
 			})
 		})
 
@@ -501,9 +509,91 @@ func TestGateway(t *testing.T) {
 			t.Run("present IPNS key from node 1", func(t *testing.T) {
 				t.Parallel()
 				node2.IPFS("name", "publish", "/ipfs/"+cidBar)
-				assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode)
-
+				assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode)
 			})
 		})
 	})
+
+	t.Run("DeserializedResponses", func(t *testing.T) {
+		type testCase struct {
+			globalValue                   config.Flag
+			gatewayValue                  config.Flag
+			deserializedGlobalStatusCode  int
+			deserializedGatewayStaticCode int
+			message                       string
+		}
+
+		setHost := func(r *http.Request) {
+			r.Host = "example.com"
+		}
+
+		withAccept := func(accept string) func(r *http.Request) {
+			return func(r *http.Request) {
+				r.Header.Set("Accept", accept)
+			}
+		}
+
+		withHostAndAccept := func(accept string) func(r *http.Request) {
+			return func(r *http.Request) {
+				setHost(r)
+				withAccept(accept)(r)
+			}
+		}
+
+		makeTest := func(test *testCase) func(t *testing.T) {
+			return func(t *testing.T) {
+				t.Parallel()
+
+				node := harness.NewT(t).NewNode().Init()
+				node.UpdateConfig(func(cfg *config.Config) {
+					cfg.Gateway.DeserializedResponses = test.globalValue
+					cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{
+						"example.com": {
+							Paths:                 []string{"/ipfs", "/ipns"},
+							DeserializedResponses: test.gatewayValue,
+						},
+					}
+				})
+				node.StartDaemon()
+
+				cidFoo := node.IPFSAddStr("foo")
+				client := node.GatewayClient()
+
+				deserializedPath := "/ipfs/" + cidFoo
+
+				blockPath := deserializedPath + "?format=raw"
+				carPath := deserializedPath + "?format=car"
+
+				// Global Check (Gateway.DeserializedResponses)
+				assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode)
+
+				assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode)
+
+				assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode)
+				assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode)
+
+				// Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses)
+				assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode)
+
+				assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode)
+				assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode)
+
+				assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode)
+				assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode)
+
+			}
+		}
+
+		for _, test := range []*testCase{
+			{config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"},
+			{config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"},
+			{config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"},
+			{config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"},
+		} {
+			t.Run(test.message, makeTest(test))
+		}
+	})
 }

test/cli/routing_dht_test.go

@@ -117,7 +117,23 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
 	})
 }
 
+func testSelfFindDHT(t *testing.T) {
+	t.Run("ipfs routing findpeer fails for self", func(t *testing.T) {
+		t.Parallel()
+		nodes := harness.NewT(t).NewNodes(1).Init()
+		nodes.ForEachPar(func(node *harness.Node) {
+			node.IPFS("config", "Routing.Type", "dht")
+		})
+
+		nodes.StartDaemons()
+
+		res := nodes[0].RunIPFS("dht", "findpeer", nodes[0].PeerID().String())
+		assert.Equal(t, 1, res.ExitCode())
+	})
+}
+
 func TestRoutingDHT(t *testing.T) {
 	testRoutingDHT(t, false)
 	testRoutingDHT(t, true)
+	testSelfFindDHT(t)
 }

test/cli/swarm_test.go

@@ -35,7 +35,7 @@ func TestSwarm(t *testing.T) {
 		res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify")
 		var output expectedOutputType
 		err := json.Unmarshal(res.Stdout.Bytes(), &output)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, 0, len(output.Peers))
 
 	})
@@ -48,7 +48,7 @@ func TestSwarm(t *testing.T) {
 		res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify")
 		var output expectedOutputType
 		err := json.Unmarshal(res.Stdout.Bytes(), &output)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		actualID := output.Peers[0].Identify.ID
 		actualPublicKey := output.Peers[0].Identify.PublicKey
 		actualAgentVersion := output.Peers[0].Identify.AgentVersion
@@ -78,12 +78,12 @@ func TestSwarm(t *testing.T) {
 		otherNodeIDResponse := otherNode.RunIPFS("id", "--enc=json")
 		var otherNodeIDOutput identifyType
 		err := json.Unmarshal(otherNodeIDResponse.Stdout.Bytes(), &otherNodeIDOutput)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify")
 
 		var output expectedOutputType
 		err = json.Unmarshal(res.Stdout.Bytes(), &output)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		outputIdentify := output.Peers[0].Identify
 
 		assert.Equal(t, outputIdentify.ID, otherNodeIDOutput.ID)

test/cli/testutils/floats.go

@@ -0,0 +1,9 @@
+package testutils
+
+func FloatTruncate(value float64, decimalPlaces int) float64 {
+	pow := 1.0
+	for i := 0; i < decimalPlaces; i++ {
+		pow *= 10.0
+	}
+	return float64(int(value*pow)) / pow
+}

test/dependencies/go.mod

@@ -2,28 +2,26 @@ module github.com/ipfs/kubo/test/dependencies
 
 go 1.18
 
+replace github.com/ipfs/kubo => ../../
+
 require (
 	github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
 	github.com/golangci/golangci-lint v1.49.0
-	github.com/ipfs/go-blockservice v0.3.0
-	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/boxo v0.10.0
+	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0
 	github.com/ipfs/go-datastore v0.6.0
-	github.com/ipfs/go-graphsync v0.14.3
-	github.com/ipfs/go-ipfs-blockstore v1.2.0
-	github.com/ipfs/go-ipfs-exchange-offline v0.2.0
+	github.com/ipfs/go-graphsync v0.14.4
 	github.com/ipfs/go-log v1.0.5
-	github.com/ipfs/go-merkledag v0.8.1
-	github.com/ipfs/go-unixfs v0.4.3
 	github.com/ipfs/hang-fds v0.1.0
 	github.com/ipfs/iptb v1.4.0
-	github.com/ipfs/iptb-plugins v0.3.0
+	github.com/ipfs/iptb-plugins v0.5.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jbenet/go-random-files v0.0.0-20190219210431-31b3f20ebded
-	github.com/libp2p/go-libp2p v0.26.4
-	github.com/multiformats/go-multiaddr v0.8.0
-	github.com/multiformats/go-multihash v0.2.1
+	github.com/libp2p/go-libp2p v0.27.5
+	github.com/multiformats/go-multiaddr v0.9.0
+	github.com/multiformats/go-multihash v0.2.3
 	gotest.tools/gotestsum v0.4.2
 )
 
@@ -36,7 +34,7 @@ require (
 	github.com/GaijinEntertainment/go-exhaustruct/v2 v2.3.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/OpenPeeDeeP/depguard v1.1.0 // indirect
-	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/alexkohler/prealloc v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/ashanbrown/forbidigo v1.3.0 // indirect
@@ -52,17 +50,17 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/charithe/durationcheck v0.0.9 // indirect
 	github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/curioswitch/go-reassign v0.1.2 // indirect
 	github.com/daixiang0/gci v0.6.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/denis-tingaikin/go-header v0.4.3 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/esimonov/ifshort v1.0.4 // indirect
 	github.com/ettle/strcase v0.1.1 // indirect
@@ -72,10 +70,12 @@ require (
 	github.com/firefart/nonamedreturns v1.0.4 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/go-critic/go-critic v0.6.4 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/go-toolsmith/astcast v1.0.0 // indirect
 	github.com/go-toolsmith/astcopy v1.0.1 // indirect
 	github.com/go-toolsmith/astequal v1.0.2 // indirect
@@ -89,7 +89,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
 	github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect
@@ -101,42 +101,39 @@ require (
 	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
+	github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
 	github.com/gostaticanalysis/comment v1.4.2 // indirect
 	github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
 	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
 	github.com/gxed/go-shellwords v1.0.3 // indirect
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hexops/gotextdiff v1.0.3 // indirect
-	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/huin/goupnp v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
-	github.com/ipfs/go-block-format v0.0.3 // indirect
-	github.com/ipfs/go-ipfs-config v0.19.0 // indirect
+	github.com/ipfs/go-block-format v0.1.2 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
-	github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect
-	github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
-	github.com/ipfs/go-ipld-format v0.4.0 // indirect
-	github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
+	github.com/ipfs/go-ipfs-files v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+	github.com/ipfs/go-ipld-format v0.5.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
 	github.com/ipfs/go-log/v2 v2.5.1 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
-	github.com/ipfs/go-peertaskqueue v0.8.0 // indirect
-	github.com/ipfs/go-unixfsnode v1.5.2 // indirect
-	github.com/ipfs/go-verifcid v0.0.1 // indirect
-	github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect
-	github.com/ipld/go-codec-dagpb v1.5.0 // indirect
+	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
+	github.com/ipfs/go-unixfsnode v1.7.1 // indirect
+	github.com/ipfs/kubo v0.16.0 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
@@ -147,9 +144,9 @@ require (
 	github.com/julz/importas v0.1.0 // indirect
 	github.com/kisielk/errcheck v1.6.2 // indirect
 	github.com/kisielk/gotool v1.0.0 // indirect
-	github.com/klauspost/compress v1.15.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.1 // indirect
-	github.com/koron/go-ssdp v0.0.3 // indirect
+	github.com/klauspost/compress v1.16.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kulti/thelper v0.6.3 // indirect
 	github.com/kunwardeep/paralleltest v1.0.6 // indirect
 	github.com/kyoh86/exportloopref v0.1.8 // indirect
@@ -159,8 +156,7 @@ require (
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
-	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-core v0.20.1 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
 	github.com/libp2p/go-nat v0.1.0 // indirect
 	github.com/libp2p/go-netroute v0.2.1 // indirect
@@ -172,15 +168,15 @@ require (
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
 	github.com/mgechev/revive v1.2.3 // indirect
-	github.com/miekg/dns v1.1.50 // indirect
+	github.com/miekg/dns v1.1.53 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moricho/tparallel v0.2.1 // indirect
@@ -189,8 +185,8 @@ require (
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.1.1 // indirect
-	github.com/multiformats/go-multicodec v0.8.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/nakabonne/nestif v0.3.1 // indirect
@@ -198,7 +194,7 @@ require (
 	github.com/nishanths/exhaustive v0.8.1 // indirect
 	github.com/nishanths/predeclared v0.2.2 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/ginkgo/v2 v2.5.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
@@ -207,19 +203,19 @@ require (
 	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/polyfloyd/go-errorlint v1.0.2 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/quasilyte/go-ruleguard v0.3.17 // indirect
 	github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5 // indirect
 	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
 	github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
-	github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
 	github.com/quic-go/quic-go v0.33.0 // indirect
 	github.com/quic-go/webtransport-go v0.5.2 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
@@ -247,7 +243,7 @@ require (
 	github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
 	github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.4.0 // indirect
 	github.com/sylvia7788/contextcheck v1.0.6 // indirect
 	github.com/tdakkota/asciicheck v0.1.1 // indirect
@@ -258,36 +254,35 @@ require (
 	github.com/tommy-muehle/go-mnd/v2 v2.5.0 // indirect
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.5 // indirect
-	github.com/urfave/cli v1.22.2 // indirect
+	github.com/urfave/cli v1.22.10 // indirect
 	github.com/uudashr/gocognit v1.0.6 // indirect
-	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect
 	github.com/yeya24/promlinter v0.2.0 // indirect
 	gitlab.com/bosi/decorder v0.2.3 // indirect
-	go.opentelemetry.io/otel v1.2.0 // indirect
-	go.opentelemetry.io/otel/trace v1.2.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/dig v1.15.0 // indirect
-	go.uber.org/fx v1.18.2 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/fx v1.19.3 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.4.0 // indirect
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/term v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/ini.v1 v1.66.6 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.3.3 // indirect
-	lukechampine.com/blake3 v1.1.7 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 	mvdan.cc/gofumpt v0.3.1 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect

test/dependencies/graphsync-get/graphsync-get.go

@@ -7,7 +7,11 @@ import (
 	"log"
 	"os"
 
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
+	blockstore "github.com/ipfs/boxo/blockstore"
+	offline "github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	uio "github.com/ipfs/boxo/ipld/unixfs/io"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
@@ -15,10 +19,6 @@ import (
 	gsimpl "github.com/ipfs/go-graphsync/impl"
 	"github.com/ipfs/go-graphsync/network"
 	"github.com/ipfs/go-graphsync/storeutil"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	offline "github.com/ipfs/go-ipfs-exchange-offline"
-	"github.com/ipfs/go-merkledag"
-	uio "github.com/ipfs/go-unixfs/io"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	basicnode "github.com/ipld/go-ipld-prime/node/basicnode"

test/sharness/t0053-dag.sh

@@ -428,40 +428,7 @@ test_expect_success "'ipfs dag put' check block size" '
     test_cmp resolve_data_exp resolve_data
   '
 
-  test_expect_success "dag stat of simple IPLD object" '
-    ipfs dag stat $NESTED_HASH > actual_stat_inner_ipld_obj &&
-    echo "Size: 8, NumBlocks: 1" > exp_stat_inner_ipld_obj &&
-    test_cmp exp_stat_inner_ipld_obj actual_stat_inner_ipld_obj &&
-    ipfs dag stat $HASH > actual_stat_ipld_obj &&
-    echo "Size: 54, NumBlocks: 2" > exp_stat_ipld_obj &&
-    test_cmp exp_stat_ipld_obj actual_stat_ipld_obj
-  '
 
-  test_expect_success "dag stat of simple UnixFS object" '
-    BASIC_UNIXFS=$(echo "1234" | ipfs add --pin=false -q) &&
-    ipfs dag stat $BASIC_UNIXFS > actual_stat_basic_unixfs &&
-    echo "Size: 13, NumBlocks: 1" > exp_stat_basic_unixfs &&
-    test_cmp exp_stat_basic_unixfs actual_stat_basic_unixfs
-  '
-
-  # The multiblock file is just 10000000 copies of the number 1
-  # As most of its data is replicated it should have a small number of blocks
-  test_expect_success "dag stat of multiblock UnixFS object" '
-    MULTIBLOCK_UNIXFS=$(printf "1%.0s" {1..10000000} | ipfs add --pin=false -q) &&
-    ipfs dag stat $MULTIBLOCK_UNIXFS > actual_stat_multiblock_unixfs &&
-    echo "Size: 302582, NumBlocks: 3" > exp_stat_multiblock_unixfs &&
-    test_cmp exp_stat_multiblock_unixfs actual_stat_multiblock_unixfs
-  '
-
-  test_expect_success "dag stat of directory of UnixFS objects" '
-    mkdir -p unixfsdir &&
-    echo "1234" > unixfsdir/small.txt
-    printf "1%.0s" {1..10000000} > unixfsdir/many1s.txt &&
-    DIRECTORY_UNIXFS=$(ipfs add -r --pin=false -Q unixfsdir) &&
-    ipfs dag stat $DIRECTORY_UNIXFS > actual_stat_directory_unixfs &&
-    echo "Size: 302705, NumBlocks: 5" > exp_stat_directory_unixfs &&
-    test_cmp exp_stat_directory_unixfs actual_stat_directory_unixfs
-  '
 }
 
 # should work offline