From bc7fd33bab35f3601f0935c988bc3a1f1944a233 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 16 Dec 2025 22:56:29 +0100 Subject: [PATCH 01/12] ci: parallelize gotest by separating test/cli into own job split the Go Test workflow into two parallel jobs: - `unit-tests`: runs unit tests (excluding test/cli) - `cli-tests`: runs test/cli end-to-end tests test/cli takes ~3 minutes (~50% of total gotest time), so running it in parallel should reduce wall-clock CI time by ~1.5-2.5 minutes. both jobs produce JUnit XML and HTML reports for consistent debugging. --- .github/workflows/gotest.yml | 75 +++++++++++++++++++++++++++++++++++- coverage/Rules.mk | 3 +- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 4e9e0b905..e9e03451e 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -14,10 +14,10 @@ concurrency: cancel-in-progress: true jobs: - go-test: + unit-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} - timeout-minutes: 20 + timeout-minutes: 15 env: TEST_DOCKER: 0 TEST_FUSE: 0 @@ -106,3 +106,74 @@ jobs: - name: Set the summary run: cat test/unit/gotest.md >> $GITHUB_STEP_SUMMARY if: failure() || success() + + cli-tests: + if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} + timeout-minutes: 15 + env: + TEST_VERBOSE: 1 + GIT_PAGER: cat + IPFS_CHECK_RCMGR_DEFAULTS: 1 + defaults: + run: + shell: bash + steps: + - name: Check out Kubo + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Install missing tools + run: sudo apt update && sudo apt install -y zsh + - name: Build kubo + run: make build + - name: Run CLI tests + env: + IPFS_PATH: ${{ runner.temp }}/ipfs-test + run: | + export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" + go test -v -json ./test/cli/... 2>&1 | tee test/cli/cli-tests.json + - name: Check for failures + run: | + if jq -s -e 'map(select(.Action == "fail")) | length > 0' test/cli/cli-tests.json > /dev/null; then + echo "CLI tests failed" + exit 1 + fi + if: success() + - name: Create JUnit XML report + uses: ipdxco/gotest-json-to-junit-xml@v1 + with: + input: test/cli/cli-tests.json + output: test/cli/cli-tests.junit.xml + if: failure() || success() + - name: Archive JUnit XML report + uses: actions/upload-artifact@v5 + with: + name: cli-tests + path: test/cli/cli-tests.junit.xml + if: failure() || success() + - name: Create HTML report + uses: ipdxco/junit-xml-to-html@v1 + with: + mode: no-frames + input: test/cli/cli-tests.junit.xml + output: test/cli/cli-tests.html + if: failure() || success() + - name: Archive HTML report + uses: actions/upload-artifact@v5 + with: + name: cli-html + path: test/cli/cli-tests.html + if: failure() || success() + - name: Create Markdown report + uses: ipdxco/junit-xml-to-html@v1 + with: + mode: summary + input: test/cli/cli-tests.junit.xml + output: test/cli/cli-tests.md + if: failure() || success() + - name: Set summary + run: cat test/cli/cli-tests.md >> $GITHUB_STEP_SUMMARY + if: failure() || success() diff --git a/coverage/Rules.mk b/coverage/Rules.mk index 48fce2856..e7ad1fa5f 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -13,7 +13,8 @@ endif .PHONY: $(d)/coverage_deps # unit tests coverage -UTESTS_$(d) := $(shell $(GOCC) list -f '{{if (or (len .TestGoFiles) (len .XTestGoFiles))}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps) +# Note: test/cli is excluded here as it runs in a separate cli-tests job +UTESTS_$(d) := $(shell $(GOCC) list -f '{{if (or (len .TestGoFiles) (len .XTestGoFiles))}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps | grep -v '/test/cli') UCOVER_$(d) := $(addsuffix .coverprofile,$(addprefix $(d)/unitcover/, $(subst /,_,$(UTESTS_$(d))))) From 15b5eb7a13c04c6dd46210e5a7f8da910e39a88e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 16 Dec 2025 23:03:15 +0100 Subject: [PATCH 02/12] ci(gotest): reduce noise on test timeout panics add GOTRACEBACK=single to show only one goroutine stack instead of all when a test timeout panic occurs. this makes CI output much cleaner when tests hang. --- .github/workflows/gotest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index e9e03451e..3fecacb7b 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -19,6 +19,7 @@ jobs: runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 15 env: + GOTRACEBACK: single # reduce noise on test timeout panics TEST_DOCKER: 0 TEST_FUSE: 0 TEST_VERBOSE: 1 @@ -112,6 +113,7 @@ jobs: runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 15 env: + GOTRACEBACK: single # reduce noise on test timeout panics TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 From 9cfa982f099dd1ad0cf25a1e011644f14f83c653 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 16 Dec 2025 23:52:09 +0100 Subject: [PATCH 03/12] fix(ci): prevent stderr from corrupting test JSON output - remove 2>&1 which mixed "go: downloading" stderr messages into JSON - add JSON validation before parsing - print failed test names for easier debugging --- .github/workflows/gotest.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 3fecacb7b..86b73e5f5 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -136,14 +136,19 @@ jobs: IPFS_PATH: ${{ runner.temp }}/ipfs-test run: | export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" - go test -v -json ./test/cli/... 2>&1 | tee test/cli/cli-tests.json + go test -v -json ./test/cli/... | tee test/cli/cli-tests.json || true - name: Check for failures run: | - if jq -s -e 'map(select(.Action == "fail")) | length > 0' test/cli/cli-tests.json > /dev/null; then - echo "CLI tests failed" + # Check JSON is valid and look for test failures + if ! jq -e . test/cli/cli-tests.json > /dev/null 2>&1; then + echo "::error::JSON output is malformed" + exit 1 + fi + if jq -s -e 'map(select(.Action == "fail")) | length > 0' test/cli/cli-tests.json > /dev/null; then + echo "CLI tests failed" + jq -s -r 'map(select(.Action == "fail")) | .[] | "FAIL: \(.Package) \(.Test // "")"' test/cli/cli-tests.json exit 1 fi - if: success() - name: Create JUnit XML report uses: ipdxco/gotest-json-to-junit-xml@v1 with: From 5100324d05c478d54cdb2dd784f886fd7f00d0f9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 01:05:45 +0100 Subject: [PATCH 04/12] ci(gotest): use gotestsum for human-readable test output - replace per-package coverage loop with single gotestsum invocation - both unit-tests and cli-tests now show human-readable output - simplified coverage collection (single -coverprofile, no gocovmerge) - clarified step names to indicate they run tests --- .github/workflows/gotest.yml | 30 ++++++++++-------------------- coverage/Rules.mk | 28 ++++++++-------------------- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 86b73e5f5..155846990 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -14,6 +14,7 @@ concurrency: cancel-in-progress: true jobs: + # Unit tests with coverage collection (uploaded to Codecov) unit-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} @@ -37,12 +38,9 @@ jobs: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - - name: 👉️ If this step failed, go to «Summary» (top left) → inspect the «Failures/Errors» table - env: - # increasing parallelism beyond 2 doesn't speed up the tests much - PARALLEL: 2 + - name: Run unit tests 👉️ Summary (top left) → «Failures/Errors» table run: | - make -j "$PARALLEL" test/unit/gotest.junit.xml && + make test/unit/gotest.junit.xml && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 @@ -108,6 +106,8 @@ jobs: run: cat test/unit/gotest.md >> $GITHUB_STEP_SUMMARY if: failure() || success() + # End-to-end integration/regression tests from test/cli + # (Go-based replacement for legacy test/sharness shell scripts) cli-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} @@ -131,24 +131,14 @@ jobs: run: sudo apt update && sudo apt install -y zsh - name: Build kubo run: make build - - name: Run CLI tests + - name: Install gotestsum + run: make test/bin/gotestsum + - name: Run CLI tests 👉️ Summary (top left) → «Failures/Errors» table env: IPFS_PATH: ${{ runner.temp }}/ipfs-test run: | - export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" - go test -v -json ./test/cli/... | tee test/cli/cli-tests.json || true - - name: Check for failures - run: | - # Check JSON is valid and look for test failures - if ! jq -e . test/cli/cli-tests.json > /dev/null 2>&1; then - echo "::error::JSON output is malformed" - exit 1 - fi - if jq -s -e 'map(select(.Action == "fail")) | length > 0' test/cli/cli-tests.json > /dev/null; then - echo "CLI tests failed" - jq -s -r 'map(select(.Action == "fail")) | .[] | "FAIL: \(.Package) \(.Test // "")"' test/cli/cli-tests.json - exit 1 - fi + export PATH="${{ github.workspace }}/cmd/ipfs:${{ github.workspace }}/test/bin:$PATH" + gotestsum --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... - name: Create JUnit XML report uses: ipdxco/gotest-json-to-junit-xml@v1 with: diff --git a/coverage/Rules.mk b/coverage/Rules.mk index e7ad1fa5f..2310489f7 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -3,30 +3,18 @@ include mk/header.mk GOCC ?= go $(d)/coverage_deps: $$(DEPS_GO) cmd/ipfs/ipfs - rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover -ifneq ($(IPFS_SKIP_COVER_BINS),1) -$(d)/coverage_deps: test/bin/gocovmerge -endif - .PHONY: $(d)/coverage_deps -# unit tests coverage -# Note: test/cli is excluded here as it runs in a separate cli-tests job -UTESTS_$(d) := $(shell $(GOCC) list -f '{{if (or (len .TestGoFiles) (len .XTestGoFiles))}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps | grep -v '/test/cli') +# unit tests coverage (excludes test/cli which runs in separate cli-tests job) +# Uses gotestsum for human-readable output while collecting coverage +UTESTS_$(d) := $(shell $(GOCC) list $(go-flags-with-tags) ./... | grep -v '/vendor' | grep -v '/Godeps' | grep -v '/test/cli') -UCOVER_$(d) := $(addsuffix .coverprofile,$(addprefix $(d)/unitcover/, $(subst /,_,$(UTESTS_$(d))))) - -$(UCOVER_$(d)): $(d)/coverage_deps ALWAYS - $(eval TMP_PKG := $(subst _,/,$(basename $(@F)))) - $(eval TMP_DEPS := $(shell $(GOCC) list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) $(TMP_PKG) | sed 's/ /\n/g' | grep ipfs/go-ipfs) $(TMP_PKG)) - $(eval TMP_DEPS_LIST := $(call join-with,$(comma),$(TMP_DEPS))) - $(GOCC) test $(go-flags-with-tags) $(GOTFLAGS) -v -covermode=atomic -json -coverpkg=$(TMP_DEPS_LIST) -coverprofile=$@ $(TMP_PKG) | tee -a test/unit/gotest.json - - -$(d)/unit_tests.coverprofile: $(UCOVER_$(d)) - gocovmerge $^ > $@ +$(d)/unit_tests.coverprofile: test/bin/gotestsum $$(DEPS_GO) + rm -f test/unit/gotest.json + gotestsum --no-color --jsonfile test/unit/gotest.json \ + -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=$@ -coverpkg=./... $(UTESTS_$(d)) TGTS_$(d) := $(d)/unit_tests.coverprofile @@ -47,7 +35,7 @@ endif export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ $(d)/sharness_tests.coverprofile: export TEST_PLUGIN=0 -$(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness +$(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test/bin/gocovmerge test_sharness (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ From 0673391d9022201c21596537c62f098afb034c6c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 01:17:50 +0100 Subject: [PATCH 05/12] ci: fix codecov uploads by adding token - add CODECOV_TOKEN to gotest.yml and sharness.yml - update codecov-action to v5.5.2 - add fail_ci_if_error: false for robustness codecov stopped receiving coverage data ~1 year ago when they started requiring tokens for public repos --- .github/workflows/gotest.yml | 4 +++- .github/workflows/sharness.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 155846990..03a91d935 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -43,11 +43,13 @@ jobs: make test/unit/gotest.junit.xml && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 if: failure() || success() with: name: unittests files: coverage/unit_tests.coverprofile + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false - name: Test kubo-as-a-library example run: | # we want to first test with the kubo version in the go.mod file diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index 67ebd208e..07ab6d61f 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -55,11 +55,13 @@ jobs: # increasing parallelism beyond 10 doesn't speed up the tests much PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }} - name: Upload coverage report - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 if: failure() || success() with: name: sharness files: kubo/coverage/sharness_tests.coverprofile + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false - name: Aggregate results run: find kubo/test/sharness/test-results -name 't*-*.sh.*.counts' | kubo/test/sharness/lib/sharness/aggregate-results.sh > kubo/test/sharness/test-results/summary.txt - name: 👉️ If this step failed, go to «Summary» (top left) → «HTML Report» → inspect the «Failures» column From 09d94888dd29884b4114af210718a22779e33908 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 01:37:58 +0100 Subject: [PATCH 06/12] refactor(make): add test_unit and test_cli targets - add `make test_unit` for unit tests with coverage (used by CI) - add `make test_cli` for CLI integration tests (used by CI) - only disable colors when CI env var is set (local dev gets colors) - remove legacy targets: test_go_test, test_go_short, test_go_race, test_go_expensive - update gotest.yml to use make targets instead of inline commands - add test artifacts to .gitignore --- .github/workflows/gotest.yml | 10 ++------ .gitignore | 5 ++++ Rules.mk | 15 ++++++------ coverage/Rules.mk | 14 +++--------- mk/golang.mk | 44 +++++++++++++++++------------------- test/unit/Rules.mk | 3 ++- 6 files changed, 40 insertions(+), 51 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 03a91d935..5d926e468 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -40,7 +40,7 @@ jobs: run: sudo apt update && sudo apt install -y zsh - name: Run unit tests 👉️ Summary (top left) → «Failures/Errors» table run: | - make test/unit/gotest.junit.xml && + make test_unit && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 @@ -131,16 +131,10 @@ jobs: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - - name: Build kubo - run: make build - - name: Install gotestsum - run: make test/bin/gotestsum - name: Run CLI tests 👉️ Summary (top left) → «Failures/Errors» table env: IPFS_PATH: ${{ runner.temp }}/ipfs-test - run: | - export PATH="${{ github.workspace }}/cmd/ipfs:${{ github.workspace }}/test/bin:$PATH" - gotestsum --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... + run: make test_cli - name: Create JUnit XML report uses: ipdxco/gotest-json-to-junit-xml@v1 with: diff --git a/.gitignore b/.gitignore index cb147456b..890870a6e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,11 @@ go-ipfs-source.tar.gz docs/examples/go-ipfs-as-a-library/example-folder/Qm* /test/sharness/t0054-dag-car-import-export-data/*.car +# test artifacts from make test_unit / test_cli +/test/unit/gotest.json +/test/unit/gotest.junit.xml +/test/cli/cli-tests.json + # ignore build output from snapcraft /ipfs_*.snap /parts diff --git a/Rules.mk b/Rules.mk index d8f16ada8..b04e3d73e 100644 --- a/Rules.mk +++ b/Rules.mk @@ -134,15 +134,14 @@ help: @echo '' @echo 'TESTING TARGETS:' @echo '' - @echo ' test - Run all tests' - @echo ' test_short - Run short go tests and short sharness tests' - @echo ' test_go_short - Run short go tests' - @echo ' test_go_test - Run all go tests' + @echo ' test - Run all tests (test_go_fmt, test_unit, test_cli, test_sharness)' + @echo ' test_short - Run fast tests (test_go_fmt, test_unit)' + @echo ' test_unit - Run unit tests with coverage (excludes test/cli)' + @echo ' test_cli - Run CLI integration tests (requires built binary)' + @echo ' test_go_fmt - Check Go source formatting' @echo ' test_go_build - Build kubo for all platforms from .github/build-platforms.yml' - @echo ' test_go_expensive - Run all go tests and build all platforms' - @echo ' test_go_race - Run go tests with the race detector enabled' - @echo ' test_go_lint - Run the `golangci-lint` vetting tool' + @echo ' test_go_lint - Run golangci-lint' @echo ' test_sharness - Run sharness tests' - @echo ' coverage - Collects coverage info from unit tests and sharness' + @echo ' coverage - Collect coverage info from unit tests and sharness' @echo .PHONY: help diff --git a/coverage/Rules.mk b/coverage/Rules.mk index 2310489f7..84a4a1887 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -7,18 +7,10 @@ $(d)/coverage_deps: $$(DEPS_GO) cmd/ipfs/ipfs .PHONY: $(d)/coverage_deps -# unit tests coverage (excludes test/cli which runs in separate cli-tests job) -# Uses gotestsum for human-readable output while collecting coverage -UTESTS_$(d) := $(shell $(GOCC) list $(go-flags-with-tags) ./... | grep -v '/vendor' | grep -v '/Godeps' | grep -v '/test/cli') +# unit tests coverage is now produced by test_unit target in mk/golang.mk +# (outputs coverage/unit_tests.coverprofile and test/unit/gotest.json) -$(d)/unit_tests.coverprofile: test/bin/gotestsum $$(DEPS_GO) - rm -f test/unit/gotest.json - gotestsum --no-color --jsonfile test/unit/gotest.json \ - -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=$@ -coverpkg=./... $(UTESTS_$(d)) - -TGTS_$(d) := $(d)/unit_tests.coverprofile - -.PHONY: $(d)/unit_tests.coverprofile +TGTS_$(d) := # sharness tests coverage $(d)/ipfs: GOTAGS += testrunmain diff --git a/mk/golang.mk b/mk/golang.mk index b50179a0a..86ef5c54d 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -41,40 +41,38 @@ define go-build $(GOCC) build $(go-flags-with-tags) -o "$@" "$(1)" endef -test_go_test: $$(DEPS_GO) - $(GOCC) test $(go-flags-with-tags) $(GOTFLAGS) ./... -.PHONY: test_go_test +# Only disable colors when running in CI (non-interactive terminal) +GOTESTSUM_NOCOLOR := $(if $(CI),--no-color,) -# Build all platforms from .github/build-platforms.yml +# Unit tests with coverage (excludes test/cli which has separate target) +# Produces JSON for CI reporting and coverage profile for Codecov +test_unit: test/bin/gotestsum $$(DEPS_GO) + rm -f test/unit/gotest.json coverage/unit_tests.coverprofile + gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list ./... | grep -v '/test/cli') +.PHONY: test_unit + +# CLI integration tests (requires built binary in PATH) +# Produces JSON for CI reporting +test_cli: cmd/ipfs/ipfs test/bin/gotestsum + rm -f test/cli/cli-tests.json + PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... +.PHONY: test_cli + +# Build kubo for all platforms from .github/build-platforms.yml test_go_build: bin/test-go-build-platforms .PHONY: test_go_build -test_go_short: GOTFLAGS += -test.short -test_go_short: test_go_test -.PHONY: test_go_short - -test_go_race: GOTFLAGS += -race -test_go_race: test_go_test -.PHONY: test_go_race - -test_go_expensive: test_go_test test_go_build -.PHONY: test_go_expensive -TEST_GO += test_go_expensive - +# Check Go source formatting test_go_fmt: bin/test-go-fmt .PHONY: test_go_fmt -TEST_GO += test_go_fmt +# Run golangci-lint (used by CI) test_go_lint: test/bin/golangci-lint golangci-lint run --timeout=3m ./... .PHONY: test_go_lint -test_go: $(TEST_GO) - -# Version check is no longer needed - go.mod enforces minimum version -.PHONY: check_go_version - +TEST_GO := test_go_fmt test_unit test_cli TEST += $(TEST_GO) -TEST_SHORT += test_go_fmt test_go_short +TEST_SHORT += test_go_fmt test_unit diff --git a/test/unit/Rules.mk b/test/unit/Rules.mk index 69404637c..915d08f9a 100644 --- a/test/unit/Rules.mk +++ b/test/unit/Rules.mk @@ -2,7 +2,8 @@ include mk/header.mk CLEAN += $(d)/gotest.json $(d)/gotest.junit.xml -$(d)/gotest.junit.xml: test/bin/gotestsum coverage/unit_tests.coverprofile +# Convert gotest.json (produced by test_unit) to JUnit XML format +$(d)/gotest.junit.xml: test/bin/gotestsum $(d)/gotest.json gotestsum --no-color --junitfile $@ --raw-command cat $(@D)/gotest.json include mk/footer.mk From 08a68a7b4cdf3664fb87559daf96189dab0a3cc8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 01:59:56 +0100 Subject: [PATCH 07/12] fix(ci): move client/rpc tests to cli-tests job client/rpc tests use test/cli/harness which requires the ipfs binary. Move them from test_unit to test_cli where the binary is built. also: - update gotestsum to v1.13.0 - simplify workflow step names --- .github/workflows/gotest.yml | 4 ++-- mk/golang.mk | 9 +++++---- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 5d926e468..04173eab2 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -38,7 +38,7 @@ jobs: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - - name: Run unit tests 👉️ Summary (top left) → «Failures/Errors» table + - name: Run unit tests run: | make test_unit && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] @@ -131,7 +131,7 @@ jobs: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - - name: Run CLI tests 👉️ Summary (top left) → «Failures/Errors» table + - name: Run CLI tests env: IPFS_PATH: ${{ runner.temp }}/ipfs-test run: make test_cli diff --git a/mk/golang.mk b/mk/golang.mk index 86ef5c54d..0bc14c87d 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -44,18 +44,19 @@ endef # Only disable colors when running in CI (non-interactive terminal) GOTESTSUM_NOCOLOR := $(if $(CI),--no-color,) -# Unit tests with coverage (excludes test/cli which has separate target) +# Unit tests with coverage (excludes packages that need ipfs binary) # Produces JSON for CI reporting and coverage profile for Codecov test_unit: test/bin/gotestsum $$(DEPS_GO) rm -f test/unit/gotest.json coverage/unit_tests.coverprofile - gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list ./... | grep -v '/test/cli') + gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list ./... | grep -v '/test/cli' | grep -v '/client/rpc') .PHONY: test_unit -# CLI integration tests (requires built binary in PATH) +# CLI/integration tests (requires built binary in PATH) +# Includes test/cli and client/rpc (which uses test/cli/harness) # Produces JSON for CI reporting test_cli: cmd/ipfs/ipfs test/bin/gotestsum rm -f test/cli/cli-tests.json - PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... + PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... ./client/rpc/... .PHONY: test_cli # Build kubo for all platforms from .github/build-platforms.yml diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 6cebaf1f7..8eac1416a 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/iptb-plugins v0.5.1 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multihash v0.2.3 - gotest.tools/gotestsum v1.12.3 + gotest.tools/gotestsum v1.13.0 ) require ( diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 46f9b2409..d107373c3 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -985,8 +985,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/gotestsum v1.12.3 h1:jFwenGJ0RnPkuKh2VzAYl1mDOJgbhobBDeL2W1iEycs= -gotest.tools/gotestsum v1.12.3/go.mod h1:Y1+e0Iig4xIRtdmYbEV7K7H6spnjc1fX4BOuUhWw2Wk= +gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= +gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI= From 6365ad78cfc07659a8b22fa4a43b6bccd045b344 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 02:09:38 +0100 Subject: [PATCH 08/12] fix(ci): use build tags when listing test packages go list needs build tags to properly exclude packages like fuse/mfs when running with TEST_FUSE=0 (nofuse tag). --- mk/golang.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/golang.mk b/mk/golang.mk index 0bc14c87d..be45ff56c 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -48,7 +48,7 @@ GOTESTSUM_NOCOLOR := $(if $(CI),--no-color,) # Produces JSON for CI reporting and coverage profile for Codecov test_unit: test/bin/gotestsum $$(DEPS_GO) rm -f test/unit/gotest.json coverage/unit_tests.coverprofile - gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list ./... | grep -v '/test/cli' | grep -v '/client/rpc') + gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list $(go-tags) ./... | grep -v '/test/cli' | grep -v '/client/rpc') .PHONY: test_unit # CLI/integration tests (requires built binary in PATH) From c2469c3634a6b5db0706da0c77cdb2a825ef62dc Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 02:29:36 +0100 Subject: [PATCH 09/12] fix(test): improve kubo-as-a-library example test - use t.Context() for automatic timeout/cancellation (Go 1.24+) - use CombinedOutput to capture stderr on failure - show full output in error messages for debugging --- docs/examples/kubo-as-a-library/main_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/examples/kubo-as-a-library/main_test.go b/docs/examples/kubo-as-a-library/main_test.go index ec34d62b1..da10b1fd7 100644 --- a/docs/examples/kubo-as-a-library/main_test.go +++ b/docs/examples/kubo-as-a-library/main_test.go @@ -7,11 +7,12 @@ import ( ) func TestExample(t *testing.T) { - out, err := exec.Command("go", "run", "main.go").Output() + cmd := exec.CommandContext(t.Context(), "go", "run", "main.go") + out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("running example (%v)", err) + t.Fatalf("running example: %v\nOutput:\n%s", err, out) } if !strings.Contains(string(out), "All done!") { - t.Errorf("example did not run successfully") + t.Errorf("example did not complete successfully\nOutput:\n%s", out) } } From 0c79aab6e60be917b68b31e9bfcaf84d0473d826 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 17 Dec 2025 02:44:57 +0100 Subject: [PATCH 10/12] fix(ci): move test/integration to cli-tests job test/integration tests need the ipfs binary, move them from test_unit to test_cli. --- mk/golang.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mk/golang.mk b/mk/golang.mk index be45ff56c..9287f6b27 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -44,19 +44,19 @@ endef # Only disable colors when running in CI (non-interactive terminal) GOTESTSUM_NOCOLOR := $(if $(CI),--no-color,) -# Unit tests with coverage (excludes packages that need ipfs binary) +# Unit tests with coverage (excludes integration test packages) # Produces JSON for CI reporting and coverage profile for Codecov test_unit: test/bin/gotestsum $$(DEPS_GO) rm -f test/unit/gotest.json coverage/unit_tests.coverprofile - gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list $(go-tags) ./... | grep -v '/test/cli' | grep -v '/client/rpc') + gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=./... $$($(GOCC) list $(go-tags) ./... | grep -v '/test/cli' | grep -v '/test/integration' | grep -v '/client/rpc') .PHONY: test_unit # CLI/integration tests (requires built binary in PATH) -# Includes test/cli and client/rpc (which uses test/cli/harness) +# Includes test/cli, test/integration, and client/rpc # Produces JSON for CI reporting test_cli: cmd/ipfs/ipfs test/bin/gotestsum rm -f test/cli/cli-tests.json - PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v ./test/cli/... ./client/rpc/... + PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v -timeout=20m ./test/cli/... ./test/integration/... ./client/rpc/... .PHONY: test_cli # Build kubo for all platforms from .github/build-platforms.yml From 55b6c15db28c3f0a30d4296b523949a5b985280a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 19 Dec 2025 19:39:42 +0100 Subject: [PATCH 11/12] fix(test): fix flaky kubo-as-a-library and GetClosestPeers tests kubo-as-a-library: use `Bootstrap()` instead of raw `Swarm().Connect()` to fix race condition between swarm connection and bitswap peer discovery. `Bootstrap()` properly integrates peers into the routing system, ensuring bitswap learns about connected peers synchronously. GetClosestPeers: simplify retry logic using `EventuallyWithT` with 10-minute timeout. tests all 4 routing types (`auto`, `autoclient`, `dht`, `dhtclient`) against real bootstrap peers with patient polling. --- docs/examples/kubo-as-a-library/main.go | 91 ++++++------------- .../delegated_routing_v1_http_server_test.go | 58 +++--------- 2 files changed, 40 insertions(+), 109 deletions(-) diff --git a/docs/examples/kubo-as-a-library/main.go b/docs/examples/kubo-as-a-library/main.go index ffa86c7f0..8edfa1aeb 100644 --- a/docs/examples/kubo-as-a-library/main.go +++ b/docs/examples/kubo-as-a-library/main.go @@ -5,17 +5,16 @@ import ( "flag" "fmt" "io" - "log" "os" "path/filepath" "strings" "sync" "time" + "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" icore "github.com/ipfs/kubo/core/coreiface" - ma "github.com/multiformats/go-multiaddr" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" @@ -68,6 +67,10 @@ func createTempRepo(swarmPort int) (string, error) { fmt.Sprintf("/ip4/0.0.0.0/udp/%d/webrtc-direct", swarmPort), } + // Disable auto-bootstrap. For this example, we manually connect only the peers we need. + // In production, you'd typically keep the default bootstrap peers to join the network. + cfg.Bootstrap = []string{} + // When creating the repository, you can define custom settings on the repository, such as enabling experimental // features (See experimental-features.md) or customizing the gateway endpoint. // To do such things, you should modify the variable `cfg`. For example: @@ -144,40 +147,6 @@ func spawnEphemeral(ctx context.Context, swarmPort int) (icore.CoreAPI, *core.Ip return api, node, err } -func connectToPeers(ctx context.Context, ipfs icore.CoreAPI, peers []string) error { - var wg sync.WaitGroup - peerInfos := make(map[peer.ID]*peer.AddrInfo, len(peers)) - for _, addrStr := range peers { - addr, err := ma.NewMultiaddr(addrStr) - if err != nil { - return err - } - pii, err := peer.AddrInfoFromP2pAddr(addr) - if err != nil { - return err - } - pi, ok := peerInfos[pii.ID] - if !ok { - pi = &peer.AddrInfo{ID: pii.ID} - peerInfos[pi.ID] = pi - } - pi.Addrs = append(pi.Addrs, pii.Addrs...) - } - - wg.Add(len(peerInfos)) - for _, peerInfo := range peerInfos { - go func(peerInfo *peer.AddrInfo) { - defer wg.Done() - err := ipfs.Swarm().Connect(ctx, *peerInfo) - if err != nil { - log.Printf("failed to connect to %s: %s", peerInfo.ID, err) - } - }(peerInfo) - } - wg.Wait() - return nil -} - func getUnixfsNode(path string) (files.Node, error) { st, err := os.Stat(path) if err != nil { @@ -224,7 +193,7 @@ func main() { // Spawn a node using a temporary path, creating a temporary repo for the run // Using port 4011 (different from nodeA's port 4010) fmt.Println("Spawning Kubo node on a temporary repo") - ipfsB, _, err := spawnEphemeral(ctx, 4011) + ipfsB, nodeB, err := spawnEphemeral(ctx, 4011) if err != nil { panic(fmt.Errorf("failed to spawn ephemeral node: %s", err)) } @@ -297,38 +266,30 @@ func main() { fmt.Printf("Got directory back from IPFS (IPFS path: %s) and wrote it to %s\n", cidDirectory.String(), outputPathDirectory) - /// --- Part IV: Getting a file from the IPFS Network + /// --- Part IV: Getting a file from another IPFS node - fmt.Println("\n-- Going to connect to a few nodes in the Network as bootstrappers --") + fmt.Println("\n-- Connecting to nodeA and fetching content via bitswap --") - // Get nodeA's address so we can fetch the file we added to it - peerAddrs, err := ipfsA.Swarm().LocalAddrs(ctx) - if err != nil { - panic(fmt.Errorf("could not get peer addresses: %s", err)) + // In production, you'd typically connect to bootstrap peers to join the IPFS network. + // autoconf.FallbackBootstrapPeers from boxo/autoconf provides well-known bootstrap peers: + // "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + // "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + // "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + // + // For this example, we only connect nodeB to nodeA to demonstrate direct bitswap. + // We use Bootstrap() instead of raw Swarm().Connect() because Bootstrap() properly + // integrates peers into the routing system and ensures bitswap learns about them. + fmt.Println("Bootstrapping nodeB to nodeA...") + nodeAPeerInfo := nodeA.Peerstore.PeerInfo(nodeA.Identity) + if err := nodeB.Bootstrap(bootstrap.BootstrapConfigWithPeers([]peer.AddrInfo{nodeAPeerInfo})); err != nil { + panic(fmt.Errorf("failed to bootstrap nodeB to nodeA: %s", err)) } - peerMa := peerAddrs[0].String() + "/p2p/" + nodeA.Identity.String() + fmt.Println("nodeB is now connected to nodeA") - bootstrapNodes := []string{ - // In production, use autoconf.FallbackBootstrapPeers from boxo/autoconf - // which includes well-known IPFS bootstrap peers like: - // "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - // "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", - // "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - - // You can add custom peers here. For example, another IPFS node: - // "/ip4/192.0.2.1/tcp/4001/p2p/QmYourPeerID...", - // "/ip4/192.0.2.1/udp/4001/quic-v1/p2p/QmYourPeerID...", - - // nodeA's address (the peer we created above that has our test file) - peerMa, - } - - fmt.Println("Connecting to peers...") - err = connectToPeers(ctx, ipfsB, bootstrapNodes) - if err != nil { - panic(fmt.Errorf("failed to connect to peers: %s", err)) - } - fmt.Println("Connected to peers") + // Since nodeB is directly connected to nodeA, bitswap can fetch the content + // without needing DHT-based provider discovery. In a production scenario where + // nodes aren't directly connected, you'd use Routing().Provide() to announce + // content to the DHT (see docs/config.md#providedhtsweepenabled for background providing). exampleCIDStr := peerCidFile.RootCid().String() diff --git a/test/cli/delegated_routing_v1_http_server_test.go b/test/cli/delegated_routing_v1_http_server_test.go index 7883fa793..51a6834e1 100644 --- a/test/cli/delegated_routing_v1_http_server_test.go +++ b/test/cli/delegated_routing_v1_http_server_test.go @@ -2,7 +2,6 @@ package cli import ( "context" - "encoding/json" "strings" "testing" "time" @@ -21,11 +20,6 @@ import ( "github.com/stretchr/testify/require" ) -// swarmPeersOutput is used to parse the JSON output of 'ipfs swarm peers --enc=json' -type swarmPeersOutput struct { - Peers []struct{} `json:"Peers"` -} - func TestRoutingV1Server(t *testing.T) { t.Parallel() @@ -224,67 +218,43 @@ func TestRoutingV1Server(t *testing.T) { } }) - t.Run("GetClosestPeers returns peers for self", func(t *testing.T) { + t.Run("GetClosestPeers returns peers", func(t *testing.T) { t.Parallel() + // Test all DHT-enabled routing types routingTypes := []string{"auto", "autoclient", "dht", "dhtclient"} for _, routingType := range routingTypes { t.Run("routing_type="+routingType, func(t *testing.T) { t.Parallel() - // Single node with DHT and real bootstrap peers node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Routing.Type = config.NewOptionalString(routingType) - // Set real bootstrap peers from boxo/autoconf cfg.Bootstrap = autoconf.FallbackBootstrapPeers }) node.StartDaemon() - // Create client before waiting so we can probe DHT readiness c, err := client.New(node.GatewayURL()) require.NoError(t, err) - // Query for closest peers to our own peer ID key := peer.ToCid(node.PeerID()) - // Wait for node to connect to bootstrap peers and populate WAN DHT routing table - minPeers := len(autoconf.FallbackBootstrapPeers) - require.EventuallyWithT(t, func(t *assert.CollectT) { - res := node.RunIPFS("swarm", "peers", "--enc=json") - var output swarmPeersOutput - err := json.Unmarshal(res.Stdout.Bytes(), &output) - assert.NoError(t, err) - peerCount := len(output.Peers) - // Wait until we have at least minPeers connected - assert.GreaterOrEqual(t, peerCount, minPeers, - "waiting for at least %d bootstrap peers, currently have %d", minPeers, peerCount) - }, 60*time.Second, time.Second) - - // Wait for DHT to be ready by probing GetClosestPeers until it succeeds - require.EventuallyWithT(t, func(t *assert.CollectT) { - probeCtx, probeCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer probeCancel() - probeIter, probeErr := c.GetClosestPeers(probeCtx, key) - if probeErr == nil { - probeIter.Close() + // Wait for WAN DHT routing table to be populated + var records []*types.PeerRecord + require.EventuallyWithT(t, func(ct *assert.CollectT) { + resultsIter, err := c.GetClosestPeers(t.Context(), key) + if !assert.NoError(ct, err) { + return } - assert.NoError(t, probeErr, "DHT should be ready to handle GetClosestPeers") - }, 2*time.Minute, 5*time.Second) + records, err = iter.ReadAllResults(resultsIter) + assert.NoError(ct, err) + }, 10*time.Minute, 5*time.Second) + require.NotEmpty(t, records, "should return peers close to own peerid") - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - resultsIter, err := c.GetClosestPeers(ctx, key) - require.NoError(t, err) + // Per IPIP-0476, GetClosestPeers returns at most 20 peers + assert.LessOrEqual(t, len(records), 20, "IPIP-0476 limits GetClosestPeers to 20 peers") - records, err := iter.ReadAllResults(resultsIter) - require.NoError(t, err) - - // Verify we got some peers back from WAN DHT - assert.NotEmpty(t, records, "should return some peers close to own peerid") - - // Verify structure of returned records for _, record := range records { assert.Equal(t, types.SchemaPeer, record.Schema) assert.NotNil(t, record.ID) From c2ead688a893b061163d78518e2c441b734c3dd0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 20 Dec 2025 00:24:38 +0100 Subject: [PATCH 12/12] fix(example): use bidirectional Swarm().Connect() for reliable bitswap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - connect nodes bidirectionally (A→B and B→A) to simulate mutual peering - mutual peering protects connection from resource manager culling - use port 0 for random available ports (avoids CI conflicts) - enable LoopbackAddressesOnLanDHT for local testing - move retry logic to test file using require.Eventually --- docs/examples/kubo-as-a-library/go.mod | 5 +- docs/examples/kubo-as-a-library/main.go | 65 +++++++++----------- docs/examples/kubo-as-a-library/main_test.go | 26 +++++--- 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 54c6efde7..389b8d044 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -10,7 +10,7 @@ require ( github.com/ipfs/boxo v0.35.3-0.20251202220026-0842ad274a0c github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.46.0 - github.com/multiformats/go-multiaddr v0.16.1 + github.com/stretchr/testify v1.11.1 ) require ( @@ -40,6 +40,7 @@ require ( github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/cskr/pubsub v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect @@ -137,6 +138,7 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.16.1 // indirect github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect @@ -169,6 +171,7 @@ require ( github.com/pion/turn/v4 v4.0.2 // indirect github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/probe-lab/go-libdht v0.4.0 // indirect github.com/prometheus/client_golang v1.23.2 // indirect diff --git a/docs/examples/kubo-as-a-library/main.go b/docs/examples/kubo-as-a-library/main.go index 8edfa1aeb..b40c6808f 100644 --- a/docs/examples/kubo-as-a-library/main.go +++ b/docs/examples/kubo-as-a-library/main.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" icore "github.com/ipfs/kubo/core/coreiface" @@ -46,7 +45,7 @@ func setupPlugins(externalPluginsPath string) error { return nil } -func createTempRepo(swarmPort int) (string, error) { +func createTempRepo() (string, error) { repoPath, err := os.MkdirTemp("", "ipfs-shell") if err != nil { return "", fmt.Errorf("failed to get temp dir: %s", err) @@ -58,15 +57,16 @@ func createTempRepo(swarmPort int) (string, error) { return "", err } - // Configure custom ports to avoid conflicts with other IPFS instances. - // This demonstrates how to customize the node's network addresses. + // Use port 0 to let the OS assign random available ports. + // This avoids conflicts with other services or parallel test runs. cfg.Addresses.Swarm = []string{ - fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", swarmPort), - fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic-v1", swarmPort), - fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic-v1/webtransport", swarmPort), - fmt.Sprintf("/ip4/0.0.0.0/udp/%d/webrtc-direct", swarmPort), + "/ip4/127.0.0.1/tcp/0", + "/ip4/127.0.0.1/udp/0/quic-v1", } + // Enable loopback addresses on LAN DHT for local testing. + cfg.Routing.LoopbackAddressesOnLanDHT = config.True + // Disable auto-bootstrap. For this example, we manually connect only the peers we need. // In production, you'd typically keep the default bootstrap peers to join the network. cfg.Bootstrap = []string{} @@ -121,8 +121,7 @@ func createNode(ctx context.Context, repoPath string) (*core.IpfsNode, error) { var loadPluginsOnce sync.Once // Spawns a node to be used just for this run (i.e. creates a tmp repo). -// The swarmPort parameter specifies the port for libp2p swarm listeners. -func spawnEphemeral(ctx context.Context, swarmPort int) (icore.CoreAPI, *core.IpfsNode, error) { +func spawnEphemeral(ctx context.Context) (icore.CoreAPI, *core.IpfsNode, error) { var onceErr error loadPluginsOnce.Do(func() { onceErr = setupPlugins("") @@ -132,7 +131,7 @@ func spawnEphemeral(ctx context.Context, swarmPort int) (icore.CoreAPI, *core.Ip } // Create a Temporary Repo - repoPath, err := createTempRepo(swarmPort) + repoPath, err := createTempRepo() if err != nil { return nil, nil, fmt.Errorf("failed to create temp repo: %s", err) } @@ -176,8 +175,7 @@ func main() { defer cancel() // Spawn a local peer using a temporary path, for testing purposes - // Using port 4010 to avoid conflict with default IPFS port 4001 - ipfsA, nodeA, err := spawnEphemeral(ctx, 4010) + ipfsA, nodeA, err := spawnEphemeral(ctx) if err != nil { panic(fmt.Errorf("failed to spawn peer node: %s", err)) } @@ -190,10 +188,9 @@ func main() { fmt.Printf("Added file to peer with CID %s\n", peerCidFile.String()) - // Spawn a node using a temporary path, creating a temporary repo for the run - // Using port 4011 (different from nodeA's port 4010) + // Spawn a second node using a temporary path fmt.Println("Spawning Kubo node on a temporary repo") - ipfsB, nodeB, err := spawnEphemeral(ctx, 4011) + ipfsB, nodeB, err := spawnEphemeral(ctx) if err != nil { panic(fmt.Errorf("failed to spawn ephemeral node: %s", err)) } @@ -270,29 +267,27 @@ func main() { fmt.Println("\n-- Connecting to nodeA and fetching content via bitswap --") + // Connect nodes bidirectionally for bitswap transfer. + // Mutual peering protects the connection from being culled by the resource manager on either side. // In production, you'd typically connect to bootstrap peers to join the IPFS network. - // autoconf.FallbackBootstrapPeers from boxo/autoconf provides well-known bootstrap peers: - // "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - // "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", - // "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - // - // For this example, we only connect nodeB to nodeA to demonstrate direct bitswap. - // We use Bootstrap() instead of raw Swarm().Connect() because Bootstrap() properly - // integrates peers into the routing system and ensures bitswap learns about them. - fmt.Println("Bootstrapping nodeB to nodeA...") - nodeAPeerInfo := nodeA.Peerstore.PeerInfo(nodeA.Identity) - if err := nodeB.Bootstrap(bootstrap.BootstrapConfigWithPeers([]peer.AddrInfo{nodeAPeerInfo})); err != nil { - panic(fmt.Errorf("failed to bootstrap nodeB to nodeA: %s", err)) + nodeAPeerInfo := peer.AddrInfo{ + ID: nodeA.Identity, + Addrs: nodeA.Peerstore.Addrs(nodeA.Identity), } - fmt.Println("nodeB is now connected to nodeA") - - // Since nodeB is directly connected to nodeA, bitswap can fetch the content - // without needing DHT-based provider discovery. In a production scenario where - // nodes aren't directly connected, you'd use Routing().Provide() to announce - // content to the DHT (see docs/config.md#providedhtsweepenabled for background providing). + nodeBPeerInfo := peer.AddrInfo{ + ID: nodeB.Identity, + Addrs: nodeB.Peerstore.Addrs(nodeB.Identity), + } + fmt.Println("Connecting nodes bidirectionally...") + if err := ipfsB.Swarm().Connect(ctx, nodeAPeerInfo); err != nil { + panic(fmt.Errorf("failed to connect nodeB to nodeA: %s", err)) + } + if err := ipfsA.Swarm().Connect(ctx, nodeBPeerInfo); err != nil { + panic(fmt.Errorf("failed to connect nodeA to nodeB: %s", err)) + } + fmt.Println("Nodes connected") exampleCIDStr := peerCidFile.RootCid().String() - fmt.Printf("Fetching a file from the network with CID %s\n", exampleCIDStr) outputPath := outputBasePath + exampleCIDStr testCID := path.FromCid(peerCidFile.RootCid()) diff --git a/docs/examples/kubo-as-a-library/main_test.go b/docs/examples/kubo-as-a-library/main_test.go index da10b1fd7..6fa03e3cb 100644 --- a/docs/examples/kubo-as-a-library/main_test.go +++ b/docs/examples/kubo-as-a-library/main_test.go @@ -4,15 +4,25 @@ import ( "os/exec" "strings" "testing" + "time" + + "github.com/stretchr/testify/require" ) func TestExample(t *testing.T) { - cmd := exec.CommandContext(t.Context(), "go", "run", "main.go") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("running example: %v\nOutput:\n%s", err, out) - } - if !strings.Contains(string(out), "All done!") { - t.Errorf("example did not complete successfully\nOutput:\n%s", out) - } + // Use Eventually to handle async bitswap timing - the peer connection + // may need a moment before bitswap can successfully retrieve blocks. + require.Eventually(t, func() bool { + cmd := exec.CommandContext(t.Context(), "go", "run", "main.go") + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("attempt failed: %v\nOutput:\n%s", err, out) + return false + } + if !strings.Contains(string(out), "All done!") { + t.Logf("example did not complete successfully\nOutput:\n%s", out) + return false + } + return true + }, 10*time.Minute, 5*time.Second, "example failed to complete successfully") }